CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\CustomField;
  13 use Civi\Api4\UserJob;
  14
  15 /**
  16  *
  17  * @package CRM
  18  * @copyright CiviCRM LLC https://civicrm.org/licensing
  19  */
  20 abstract class CRM_Import_Parser {
  21   /**
  22    * Settings
  23    */
  24   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  25
  26   /**
  27    * Return codes
  28    */
  29   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  30
  31   /**
  32    * Parser modes
  33    */
  34   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  35
  36   /**
  37    * Codes for duplicate record handling
  38    */
  39   const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  40
  41   /**
  42    * Contact types
  43    */
  44   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  45
  46   /**
  47    * User job id.
  48    *
  49    * This is the primary key of the civicrm_user_job table which is used to
  50    * track the import.
  51    *
  52    * @var int
  53    */
  54   protected $userJobID;
  55
  56   /**
  57    * The user job in use.
  58    *
  59    * @var array
  60    */
  61   protected $userJob;
  62
  63   /**
  64    * Potentially ambiguous options.
  65    *
  66    * For example 'UT' is a state in more than one country.
  67    *
  68    * @var array
  69    */
  70   protected $ambiguousOptions = [];
  71
  72   /**
  73    * States to country mapping.
  74    *
  75    * @var array
  76    */
  77   protected $statesByCountry = [];
  78
  79   /**
  80    * @return int|null
  81    */
  82   public function getUserJobID(): ?int {
  83     return $this->userJobID;
  84   }
  85
  86   /**
  87    * Ids of contacts created this iteration.
  88    *
  89    * @var array
  90    */
  91   protected $createdContacts = [];
  92
  93   /**
  94    * Set user job ID.
  95    *
  96    * @param int $userJobID
  97    *
  98    * @return self
  99    */
 100   public function setUserJobID(int $userJobID): self {
 101     $this->userJobID = $userJobID;
 102     return $this;
 103   }
 104
 105   /**
 106    * Countries that the site is restricted to
 107    *
 108    * @var array|false
 109    */
 110   private $availableCountries;
 111
 112   /**
 113    * Get User Job.
 114    *
 115    * API call to retrieve the userJob row.
 116    *
 117    * @return array
 118    *
 119    * @throws \API_Exception
 120    */
 121   protected function getUserJob(): array {
 122     if (empty($this->userJob)) {
 123       $this->userJob = UserJob::get()
 124         ->addWhere('id', '=', $this->getUserJobID())
 125         ->execute()
 126         ->first();
 127     }
 128     return $this->userJob;
 129   }
 130
 131   /**
 132    * Get the relevant datasource object.
 133    *
 134    * @return \CRM_Import_DataSource|null
 135    *
 136    * @throws \API_Exception
 137    */
 138   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 139     $className = $this->getSubmittedValue('dataSource');
 140     if ($className) {
 141       /* @var CRM_Import_DataSource $dataSource */
 142       return new $className($this->getUserJobID());
 143     }
 144     return NULL;
 145   }
 146
 147   /**
 148    * Get the submitted value, as stored on the user job.
 149    *
 150    * @param string $fieldName
 151    *
 152    * @return mixed
 153    *
 154    * @noinspection PhpDocMissingThrowsInspection
 155    * @noinspection PhpUnhandledExceptionInspection
 156    */
 157   protected function getSubmittedValue(string $fieldName) {
 158     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 159   }
 160
 161   /**
 162    * Has the import completed.
 163    *
 164    * @return bool
 165    *
 166    * @throws \API_Exception
 167    * @throws \CRM_Core_Exception
 168    */
 169   public function isComplete() :bool {
 170     return $this->getDataSourceObject()->isCompleted();
 171   }
 172
 173   /**
 174    * Get configured contact type.
 175    *
 176    * @throws \API_Exception
 177    */
 178   protected function getContactType() {
 179     if (!$this->_contactType) {
 180       $contactTypeMapping = [
 181         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 182         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 183         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 184       ];
 185       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 186     }
 187     return $this->_contactType;
 188   }
 189
 190   /**
 191    * Get configured contact type.
 192    *
 193    * @return string|null
 194    *
 195    * @throws \API_Exception
 196    */
 197   public function getContactSubType() {
 198     if (!$this->_contactSubType) {
 199       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 200     }
 201     return $this->_contactSubType;
 202   }
 203
 204   /**
 205    * Total number of non empty lines
 206    * @var int
 207    */
 208   protected $_totalCount;
 209
 210   /**
 211    * Running total number of valid lines
 212    * @var int
 213    */
 214   protected $_validCount;
 215
 216   /**
 217    * Running total number of invalid rows
 218    * @var int
 219    */
 220   protected $_invalidRowCount;
 221
 222   /**
 223    * Maximum number of non-empty/comment lines to process
 224    *
 225    * @var int
 226    */
 227   protected $_maxLinesToProcess;
 228
 229   /**
 230    * Array of error lines, bounded by MAX_ERROR
 231    * @var array
 232    */
 233   protected $_errors;
 234
 235   /**
 236    * Total number of duplicate (from database) lines
 237    * @var int
 238    */
 239   protected $_duplicateCount;
 240
 241   /**
 242    * Array of duplicate lines
 243    * @var array
 244    */
 245   protected $_duplicates;
 246
 247   /**
 248    * Maximum number of warnings to store
 249    * @var int
 250    */
 251   protected $_maxWarningCount = self::MAX_WARNINGS;
 252
 253   /**
 254    * Array of warning lines, bounded by MAX_WARNING
 255    * @var array
 256    */
 257   protected $_warnings;
 258
 259   /**
 260    * Array of all the fields that could potentially be part
 261    * of this import process
 262    * @var array
 263    */
 264   protected $_fields;
 265
 266   /**
 267    * Metadata for all available fields, keyed by unique name.
 268    *
 269    * This is intended to supercede $_fields which uses a special sauce format which
 270    * importableFieldsMetadata uses the standard getfields type format.
 271    *
 272    * @var array
 273    */
 274   protected $importableFieldsMetadata = [];
 275
 276   /**
 277    * Get metadata for all importable fields in std getfields style format.
 278    *
 279    * @return array
 280    */
 281   public function getImportableFieldsMetadata(): array {
 282     return $this->importableFieldsMetadata;
 283   }
 284
 285   /**
 286    * Set metadata for all importable fields in std getfields style format.
 287    *
 288    * @param array $importableFieldsMetadata
 289    */
 290   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 291     $this->importableFieldsMetadata = $importableFieldsMetadata;
 292   }
 293
 294   /**
 295    * Gets the fields available for importing in a key-name, title format.
 296    *
 297    * @return array
 298    *   eg. ['first_name' => 'First Name'.....]
 299    *
 300    * @throws \API_Exception
 301    *
 302    * @todo - we are constructing the metadata before we
 303    * have set the contact type so we re-do it here.
 304    *
 305    * Once we have cleaned up the way the mapper is handled
 306    * we can ditch all the existing _construct parameters in favour
 307    * of just the userJobID - there are current open PRs towards this end.
 308    */
 309   public function getAvailableFields(): array {
 310     $this->setFieldMetadata();
 311     $return = [];
 312     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 313       if ($name === 'id' && $this->isSkipDuplicates()) {
 314         // Duplicates are being skipped so id matching is not availble.
 315         continue;
 316       }
 317       $return[$name] = $field['html']['label'] ?? $field['title'];
 318     }
 319     return $return;
 320   }
 321
 322   /**
 323    * Did the user specify duplicates should be skipped and not imported.
 324    *
 325    * @return bool
 326    *
 327    * @throws \API_Exception
 328    */
 329   protected function isSkipDuplicates(): bool {
 330     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 331   }
 332
 333   /**
 334    * Did the user specify duplicates should be filled with missing data.
 335    *
 336    * @return bool
 337    */
 338   protected function isFillDuplicates(): bool {
 339     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_FILL;
 340   }
 341
 342   /**
 343    * Array of the fields that are actually part of the import process
 344    * the position in the array also dictates their position in the import
 345    * file
 346    * @var array
 347    */
 348   protected $_activeFields = [];
 349
 350   /**
 351    * Cache the count of active fields
 352    *
 353    * @var int
 354    */
 355   protected $_activeFieldCount;
 356
 357   /**
 358    * Cache of preview rows
 359    *
 360    * @var array
 361    */
 362   protected $_rows;
 363
 364   /**
 365    * Filename of error data
 366    *
 367    * @var string
 368    */
 369   protected $_errorFileName;
 370
 371   /**
 372    * Filename of duplicate data
 373    *
 374    * @var string
 375    */
 376   protected $_duplicateFileName;
 377
 378   /**
 379    * Contact type
 380    *
 381    * @var string
 382    */
 383   public $_contactType;
 384
 385   /**
 386    * @param string $contactType
 387    *
 388    * @return CRM_Import_Parser
 389    */
 390   public function setContactType(string $contactType): CRM_Import_Parser {
 391     $this->_contactType = $contactType;
 392     return $this;
 393   }
 394
 395   /**
 396    * Contact sub-type
 397    *
 398    * @var int|null
 399    */
 400   public $_contactSubType;
 401
 402   /**
 403    * @param int|null $contactSubType
 404    *
 405    * @return self
 406    */
 407   public function setContactSubType(?int $contactSubType): self {
 408     $this->_contactSubType = $contactSubType;
 409     return $this;
 410   }
 411
 412   /**
 413    * Class constructor.
 414    */
 415   public function __construct() {
 416     $this->_maxLinesToProcess = 0;
 417   }
 418
 419   /**
 420    * Set and validate field values.
 421    *
 422    * @param array $elements
 423    *   array.
 424    */
 425   public function setActiveFieldValues($elements): void {
 426     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 427     for ($i = 0; $i < $maxCount; $i++) {
 428       $this->_activeFields[$i]->setValue($elements[$i]);
 429     }
 430
 431     // reset all the values that we did not have an equivalent import element
 432     for (; $i < $this->_activeFieldCount; $i++) {
 433       $this->_activeFields[$i]->resetValue();
 434     }
 435   }
 436
 437   /**
 438    * Format the field values for input to the api.
 439    *
 440    * @return array
 441    *   (reference) associative array of name/value pairs
 442    */
 443   public function &getActiveFieldParams() {
 444     $params = [];
 445     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 446       if (isset($this->_activeFields[$i]->_value)
 447         && !isset($params[$this->_activeFields[$i]->_name])
 448         && !isset($this->_activeFields[$i]->_related)
 449       ) {
 450
 451         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 452       }
 453     }
 454     return $params;
 455   }
 456
 457   /**
 458    * Add progress bar to the import process. Calculates time remaining, status etc.
 459    *
 460    * @param $statusID
 461    *   status id of the import process saved in $config->uploadDir.
 462    * @param bool $startImport
 463    *   True when progress bar is to be initiated.
 464    * @param $startTimestamp
 465    *   Initial timestamp when the import was started.
 466    * @param $prevTimestamp
 467    *   Previous timestamp when this function was last called.
 468    * @param $totalRowCount
 469    *   Total number of rows in the import file.
 470    *
 471    * @return NULL|$currTimestamp
 472    */
 473   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 474     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 475
 476     if ($startImport) {
 477       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 478       //do not force the browser to display the save dialog, CRM-7640
 479       $contents = json_encode([0, $status]);
 480       file_put_contents($statusFile, $contents);
 481     }
 482     else {
 483       $rowCount = $this->_rowCount ?? $this->_lineCount;
 484       $currTimestamp = time();
 485       $time = ($currTimestamp - $prevTimestamp);
 486       $recordsLeft = $totalRowCount - $rowCount;
 487       if ($recordsLeft < 0) {
 488         $recordsLeft = 0;
 489       }
 490       $estimatedTime = ($recordsLeft / 50) * $time;
 491       $estMinutes = floor($estimatedTime / 60);
 492       $timeFormatted = '';
 493       if ($estMinutes > 1) {
 494         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 495         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 496       }
 497       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 498       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 499       $statusMsg = ts('%1 of %2 records - %3 remaining',
 500         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 501       );
 502       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 503       $contents = json_encode([$processedPercent, $status]);
 504
 505       file_put_contents($statusFile, $contents);
 506       return $currTimestamp;
 507     }
 508   }
 509
 510   /**
 511    * @return array
 512    */
 513   public function getSelectValues(): array {
 514     $values = [];
 515     foreach ($this->_fields as $name => $field) {
 516       $values[$name] = $field->_title;
 517     }
 518     return $values;
 519   }
 520
 521   /**
 522    * @return array
 523    */
 524   public function getSelectTypes() {
 525     $values = [];
 526     // This is only called from the MapField form in isolation now,
 527     // so we need to set the metadata.
 528     $this->init();
 529     foreach ($this->_fields as $name => $field) {
 530       if (isset($field->_hasLocationType)) {
 531         $values[$name] = $field->_hasLocationType;
 532       }
 533     }
 534     return $values;
 535   }
 536
 537   /**
 538    * @return array
 539    */
 540   public function getHeaderPatterns(): array {
 541     $values = [];
 542     foreach ($this->_fields as $name => $field) {
 543       if (isset($field->_headerPattern)) {
 544         $values[$name] = $field->_headerPattern;
 545       }
 546     }
 547     return $values;
 548   }
 549
 550   /**
 551    * @return array
 552    */
 553   public function getDataPatterns():array {
 554     $values = [];
 555     foreach ($this->_fields as $name => $field) {
 556       $values[$name] = $field->_dataPattern;
 557     }
 558     return $values;
 559   }
 560
 561   /**
 562    * Remove single-quote enclosures from a value array (row).
 563    *
 564    * @param array $values
 565    * @param string $enclosure
 566    *
 567    * @return void
 568    */
 569   public static function encloseScrub(&$values, $enclosure = "'") {
 570     if (empty($values)) {
 571       return;
 572     }
 573
 574     foreach ($values as $k => $v) {
 575       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 576     }
 577   }
 578
 579   /**
 580    * Setter function.
 581    *
 582    * @param int $max
 583    *
 584    * @return void
 585    */
 586   public function setMaxLinesToProcess($max) {
 587     $this->_maxLinesToProcess = $max;
 588   }
 589
 590   /**
 591    * Validate that we have the required fields to create the contact or find it to update.
 592    *
 593    * Note that the users duplicate selection affects this as follows
 594    * - if they did not select an update variant then the id field is not
 595    *   permitted in the mapping - so we can assume the presence of id means
 596    *   we should use it
 597    * - the external_identifier field is valid in place of the other fields
 598    *   when they have chosen update or fill - in this case we are only looking
 599    *   to update an existing contact.
 600    *
 601    * @param string $contactType
 602    * @param array $params
 603    * @param bool $isPermitExistingMatchFields
 604    *   True if the it is enough to have fields which will enable us to find
 605    *   an existing contact (eg. external_identifier).
 606    * @param string $prefixString
 607    *   String to include in the exception (e.g '(Child of)' if we are validating
 608    *   a related contact.
 609    *
 610    * @return void
 611    * @throws \CRM_Core_Exception
 612    */
 613   protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void {
 614     if (!empty($params['id'])) {
 615       return;
 616     }
 617     $requiredFields = [
 618       'Individual' => [
 619         'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
 620         'email' => ts('Email Address'),
 621       ],
 622       'Organization' => ['organization_name' => ts('Organization Name')],
 623       'Household' => ['household_name' => ts('Household Name')],
 624     ][$contactType];
 625     if ($isPermitExistingMatchFields) {
 626       $requiredFields['external_identifier'] = ts('External Identifier');
 627       // Historically just an email has been accepted as it is 'usually good enough'
 628       // for a dedupe rule look up - but really this is a stand in for
 629       // whatever is needed to find an existing matching contact using the
 630       // specified dedupe rule (or the default Unsupervised if not specified).
 631       $requiredFields['email'] = ts('Email Address');
 632     }
 633     $this->validateRequiredFields($requiredFields, $params, $prefixString);
 634   }
 635
 636   protected function doPostImportActions() {
 637     $userJob = $this->getUserJob();
 638     $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
 639     $actions = $userJob['metadata']['post_actions'] ?? [];
 640     if (!empty($actions['group'])) {
 641       $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']);
 642       foreach ($actions['group'] as $groupID) {
 643         $summaryInfo['groups'][$groupID]['added'] += $groupAdditions[$groupID]['added'];
 644         $summaryInfo['groups'][$groupID]['notAdded'] += $groupAdditions[$groupID]['notAdded'];
 645       }
 646     }
 647     if (!empty($actions['tag'])) {
 648       $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts, $actions['tag']);
 649       foreach ($actions['tag'] as $tagID) {
 650         $summaryInfo['tags'][$tagID]['added'] += $tagAdditions[$tagID]['added'];
 651         $summaryInfo['tags'][$tagID]['notAdded'] += $tagAdditions[$tagID]['notAdded'];
 652       }
 653     }
 654
 655     $this->userJob['metadata']['summary_info'] = $summaryInfo;
 656     UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute();
 657   }
 658
 659   public function queue() {
 660     $dataSource = $this->getDataSourceObject();
 661     $totalRowCount = $totalRows = $dataSource->getRowCount(['new']);
 662     $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']);
 663     $offset = 0;
 664     $batchSize = 5;
 665     while ($totalRows > 0) {
 666       if ($totalRows < $batchSize) {
 667         $batchSize = $totalRows;
 668       }
 669       $task = new CRM_Queue_Task(
 670         [get_class($this), 'runImport'],
 671         ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize],
 672         ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount])
 673       );
 674       $queue->createItem($task);
 675       $totalRows -= $batchSize;
 676       $offset += $batchSize;
 677     }
 678
 679   }
 680
 681   /**
 682    * Add imported contacts to groups.
 683    *
 684    * @param array $contactIDs
 685    * @param array $groups
 686    *
 687    * @return array
 688    */
 689   private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array {
 690     $groupAdditions = [];
 691     foreach ($groups as $groupID) {
 692       // @todo - this function has been in use historically but it does not seem
 693       // to add much efficiency of get + create api calls
 694       // and it doesn't give enough control over cache flushing for smaller batches.
 695       // Note that the import updates a lot of enities & checking & updating the group
 696       // shouldn't add much performance wise. However, cache flushing will
 697       $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIDs, $groupID);
 698       $groupAdditions[$groupID] = [
 699         'added' => (int) $addCount[1],
 700         'notAdded' => (int) $addCount[2],
 701       ];
 702     }
 703     return $groupAdditions;
 704   }
 705
 706   /**
 707    * Tag imported contacts.
 708    *
 709    * @param array $contactIDs
 710    * @param array $tags
 711    *
 712    * @return array
 713    */
 714   private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) {
 715     $tagAdditions = [];
 716     foreach ($tags as $tagID) {
 717       // @todo - this function has been in use historically but it does not seem
 718       // to add much efficiency of get + create api calls
 719       // and it doesn't give enough control over cache flushing for smaller batches.
 720       // Note that the import updates a lot of enities & checking & updating the group
 721       // shouldn't add much performance wise. However, cache flushing will
 722       $outcome = CRM_Core_BAO_EntityTag::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE);
 723       $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]];
 724     }
 725     return $tagAdditions;
 726   }
 727
 728   /**
 729    * Determines the file extension based on error code.
 730    *
 731    * @var int $type error code constant
 732    * @return string
 733    */
 734   public static function errorFileName($type) {
 735     $fileName = NULL;
 736     if (empty($type)) {
 737       return $fileName;
 738     }
 739
 740     $config = CRM_Core_Config::singleton();
 741     $fileName = $config->uploadDir . "sqlImport";
 742     switch ($type) {
 743       case self::ERROR:
 744         $fileName .= '.errors';
 745         break;
 746
 747       case self::DUPLICATE:
 748         $fileName .= '.duplicates';
 749         break;
 750
 751       case self::NO_MATCH:
 752         $fileName .= '.mismatch';
 753         break;
 754
 755       case self::UNPARSED_ADDRESS_WARNING:
 756         $fileName .= '.unparsedAddress';
 757         break;
 758     }
 759
 760     return $fileName;
 761   }
 762
 763   /**
 764    * Determines the file name based on error code.
 765    *
 766    * @var $type error code constant
 767    * @return string
 768    */
 769   public static function saveFileName($type) {
 770     $fileName = NULL;
 771     if (empty($type)) {
 772       return $fileName;
 773     }
 774     switch ($type) {
 775       case self::ERROR:
 776         $fileName = 'Import_Errors.csv';
 777         break;
 778
 779       case self::DUPLICATE:
 780         $fileName = 'Import_Duplicates.csv';
 781         break;
 782
 783       case self::NO_MATCH:
 784         $fileName = 'Import_Mismatch.csv';
 785         break;
 786
 787       case self::UNPARSED_ADDRESS_WARNING:
 788         $fileName = 'Import_Unparsed_Address.csv';
 789         break;
 790     }
 791
 792     return $fileName;
 793   }
 794
 795   /**
 796    * Check if contact is a duplicate .
 797    *
 798    * @param array $formatValues
 799    *
 800    * @return array
 801    */
 802   protected function checkContactDuplicate(&$formatValues) {
 803     //retrieve contact id using contact dedupe rule
 804     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->getContactType();
 805     $formatValues['version'] = 3;
 806     require_once 'CRM/Utils/DeprecatedUtils.php';
 807     $params = $formatValues;
 808     static $cIndieFields = NULL;
 809     static $defaultLocationId = NULL;
 810
 811     $contactType = $params['contact_type'];
 812     if ($cIndieFields == NULL) {
 813       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 814       $cIndieFields = $cTempIndieFields;
 815
 816       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 817
 818       // set the value to default location id else set to 1
 819       if (!$defaultLocationId = (int) $defaultLocation->id) {
 820         $defaultLocationId = 1;
 821       }
 822     }
 823
 824     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 825
 826     $contactFormatted = [];
 827     foreach ($params as $key => $field) {
 828       if ($field == NULL || $field === '') {
 829         continue;
 830       }
 831       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 832       // instead of soft credit contact.
 833       if (is_array($field) && $key !== "soft_credit") {
 834         foreach ($field as $value) {
 835           $break = FALSE;
 836           if (is_array($value)) {
 837             foreach ($value as $name => $testForEmpty) {
 838               if ($name !== 'phone_type' &&
 839                 ($testForEmpty === '' || $testForEmpty == NULL)
 840               ) {
 841                 $break = TRUE;
 842                 break;
 843               }
 844             }
 845           }
 846           else {
 847             $break = TRUE;
 848           }
 849           if (!$break) {
 850             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 851           }
 852         }
 853         continue;
 854       }
 855
 856       $value = [$key => $field];
 857
 858       // check if location related field, then we need to add primary location type
 859       if (in_array($key, $locationFields)) {
 860         $value['location_type_id'] = $defaultLocationId;
 861       }
 862       elseif (array_key_exists($key, $cIndieFields)) {
 863         $value['contact_type'] = $contactType;
 864       }
 865
 866       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 867     }
 868
 869     $contactFormatted['contact_type'] = $contactType;
 870
 871     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 872   }
 873
 874   /**
 875    * This function adds the contact variable in $values to the
 876    * parameter list $params.  For most cases, $values should have length 1.  If
 877    * the variable being added is a child of Location, a location_type_id must
 878    * also be included.  If it is a child of phone, a phone_type must be included.
 879    *
 880    * @param array $values
 881    *   The variable(s) to be added.
 882    * @param array $params
 883    *   The structured parameter list.
 884    *
 885    * @return bool|CRM_Utils_Error
 886    */
 887   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 888     // @todo - like most functions in import ... most of this is cruft....
 889     // Crawl through the possible classes:
 890     // Contact
 891     //      Individual
 892     //      Household
 893     //      Organization
 894     //          Location
 895     //              Address
 896     //              Email
 897     //              Phone
 898     //              IM
 899     //      Note
 900     //      Custom
 901
 902     // Cache the various object fields
 903     static $fields = NULL;
 904
 905     if ($fields == NULL) {
 906       $fields = [];
 907     }
 908
 909     // first add core contact values since for other Civi modules they are not added
 910     require_once 'CRM/Contact/BAO/Contact.php';
 911     $contactFields = CRM_Contact_DAO_Contact::fields();
 912     _civicrm_api3_store_values($contactFields, $values, $params);
 913
 914     if (isset($values['contact_type'])) {
 915       // we're an individual/household/org property
 916
 917       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 918
 919       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 920       return TRUE;
 921     }
 922
 923     if (isset($values['individual_prefix'])) {
 924       if (!empty($params['prefix_id'])) {
 925         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 926         $params['prefix'] = $prefixes[$params['prefix_id']];
 927       }
 928       else {
 929         $params['prefix'] = $values['individual_prefix'];
 930       }
 931       return TRUE;
 932     }
 933
 934     if (isset($values['individual_suffix'])) {
 935       if (!empty($params['suffix_id'])) {
 936         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 937         $params['suffix'] = $suffixes[$params['suffix_id']];
 938       }
 939       else {
 940         $params['suffix'] = $values['individual_suffix'];
 941       }
 942       return TRUE;
 943     }
 944
 945     if (isset($values['gender'])) {
 946       if (!empty($params['gender_id'])) {
 947         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 948         $params['gender'] = $genders[$params['gender_id']];
 949       }
 950       else {
 951         $params['gender'] = $values['gender'];
 952       }
 953       return TRUE;
 954     }
 955
 956     // format the website params.
 957     if (!empty($values['url'])) {
 958       static $websiteFields;
 959       if (!is_array($websiteFields)) {
 960         require_once 'CRM/Core/DAO/Website.php';
 961         $websiteFields = CRM_Core_DAO_Website::fields();
 962       }
 963       if (!array_key_exists('website', $params) ||
 964         !is_array($params['website'])
 965       ) {
 966         $params['website'] = [];
 967       }
 968
 969       $websiteCount = count($params['website']);
 970       _civicrm_api3_store_values($websiteFields, $values,
 971         $params['website'][++$websiteCount]
 972       );
 973
 974       return TRUE;
 975     }
 976
 977     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 978     if (!empty($values['location_type_id'])) {
 979       static $fields = NULL;
 980       if ($fields == NULL) {
 981         $fields = [];
 982       }
 983
 984       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 985         $name = strtolower($block);
 986         if (!array_key_exists($name, $values)) {
 987           continue;
 988         }
 989
 990         if ($name === 'phone_ext') {
 991           $block = 'Phone';
 992         }
 993
 994         // block present in value array.
 995         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 996           $params[$name] = [];
 997         }
 998
 999         if (!array_key_exists($block, $fields)) {
1000           $className = "CRM_Core_DAO_$block";
1001           $fields[$block] =& $className::fields();
1002         }
1003
1004         $blockCnt = count($params[$name]);
1005
1006         // copy value to dao field name.
1007         if ($name == 'im') {
1008           $values['name'] = $values[$name];
1009         }
1010
1011         _civicrm_api3_store_values($fields[$block], $values,
1012           $params[$name][++$blockCnt]
1013         );
1014
1015         if (empty($params['id']) && ($blockCnt == 1)) {
1016           $params[$name][$blockCnt]['is_primary'] = TRUE;
1017         }
1018
1019         // we only process single block at a time.
1020         return TRUE;
1021       }
1022
1023       // handle address fields.
1024       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1025         $params['address'] = [];
1026       }
1027
1028       $addressCnt = 1;
1029       foreach ($params['address'] as $cnt => $addressBlock) {
1030         if (CRM_Utils_Array::value('location_type_id', $values) ==
1031           CRM_Utils_Array::value('location_type_id', $addressBlock)
1032         ) {
1033           $addressCnt = $cnt;
1034           break;
1035         }
1036         $addressCnt++;
1037       }
1038
1039       if (!array_key_exists('Address', $fields)) {
1040         $fields['Address'] = CRM_Core_DAO_Address::fields();
1041       }
1042
1043       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1044       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1045       // the address in CRM_Core_BAO_Address::create method
1046       if (!empty($values['location_type_id'])) {
1047         static $customFields = [];
1048         if (empty($customFields)) {
1049           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1050         }
1051         // make a copy of values, as we going to make changes
1052         $newValues = $values;
1053         foreach ($values as $key => $val) {
1054           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1055           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1056             // mark an entry in fields array since we want the value of custom field to be copied
1057             $fields['Address'][$key] = NULL;
1058
1059             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1060             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1061               $mulValues = explode(',', $val);
1062               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1063               $newValues[$key] = [];
1064               foreach ($mulValues as $v1) {
1065                 foreach ($customOption as $v2) {
1066                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1067                     (strtolower($v2['value']) == strtolower(trim($v1)))
1068                   ) {
1069                     if ($htmlType == 'CheckBox') {
1070                       $newValues[$key][$v2['value']] = 1;
1071                     }
1072                     else {
1073                       $newValues[$key][] = $v2['value'];
1074                     }
1075                   }
1076                 }
1077               }
1078             }
1079           }
1080         }
1081         // consider new values
1082         $values = $newValues;
1083       }
1084
1085       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1086
1087       $addressFields = [
1088         'county',
1089         'country',
1090         'state_province',
1091         'supplemental_address_1',
1092         'supplemental_address_2',
1093         'supplemental_address_3',
1094         'StateProvince.name',
1095       ];
1096
1097       foreach ($addressFields as $field) {
1098         if (array_key_exists($field, $values)) {
1099           if (!array_key_exists('address', $params)) {
1100             $params['address'] = [];
1101           }
1102           $params['address'][$addressCnt][$field] = $values[$field];
1103         }
1104       }
1105
1106       if ($addressCnt == 1) {
1107
1108         $params['address'][$addressCnt]['is_primary'] = TRUE;
1109       }
1110       return TRUE;
1111     }
1112
1113     if (isset($values['note'])) {
1114       // add a note field
1115       if (!isset($params['note'])) {
1116         $params['note'] = [];
1117       }
1118       $noteBlock = count($params['note']) + 1;
1119
1120       $params['note'][$noteBlock] = [];
1121       if (!isset($fields['Note'])) {
1122         $fields['Note'] = CRM_Core_DAO_Note::fields();
1123       }
1124
1125       // get the current logged in civicrm user
1126       $session = CRM_Core_Session::singleton();
1127       $userID = $session->get('userID');
1128
1129       if ($userID) {
1130         $values['contact_id'] = $userID;
1131       }
1132
1133       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1134
1135       return TRUE;
1136     }
1137
1138     // Check for custom field values
1139
1140     if (empty($fields['custom'])) {
1141       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1142         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1143       );
1144     }
1145
1146     foreach ($values as $key => $value) {
1147       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1148         // check if it's a valid custom field id
1149
1150         if (!array_key_exists($customFieldID, $fields['custom'])) {
1151           return civicrm_api3_create_error('Invalid custom field ID');
1152         }
1153         else {
1154           $params[$key] = $value;
1155         }
1156       }
1157     }
1158   }
1159
1160   /**
1161    * Parse a field which could be represented by a label or name value rather than the DB value.
1162    *
1163    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1164    *
1165    * but if not available then see if we have a label that can be converted to a name.
1166    *
1167    * @param string|int|null $submittedValue
1168    * @param array $fieldSpec
1169    *   Metadata for the field
1170    *
1171    * @return mixed
1172    */
1173   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1174     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1175     if (!isset($fieldSpec['bao'])) {
1176       return $submittedValue;
1177     }
1178     /* @var \CRM_Core_DAO $bao */
1179     $bao = $fieldSpec['bao'];
1180     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1181     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1182     if (isset($nameOptions[$submittedValue])) {
1183       return $submittedValue;
1184     }
1185     if (in_array($submittedValue, $nameOptions)) {
1186       return array_search($submittedValue, $nameOptions, TRUE);
1187     }
1188
1189     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1190     if (isset($labelOptions[$submittedValue])) {
1191       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1192     }
1193     return '';
1194   }
1195
1196   /**
1197    * This is code extracted from 4 places where this exact snippet was being duplicated.
1198    *
1199    * FIXME: Extracting this was a first step, but there's also
1200    *  1. Inconsistency in the way other select options are handled.
1201    *     Contribution adds handling for Select/Radio/Autocomplete
1202    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1203    *     Membership is missing all of it
1204    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1205    *
1206    * @param $customFieldID
1207    * @param $value
1208    * @param $fieldType
1209    * @return array
1210    */
1211   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1212     $mulValues = explode(',', $value);
1213     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1214     $values = [];
1215     foreach ($mulValues as $v1) {
1216       foreach ($customOption as $customValueID => $customLabel) {
1217         $customValue = $customLabel['value'];
1218         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1219           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1220         ) {
1221           $values[] = $customValue;
1222         }
1223       }
1224     }
1225     return $values;
1226   }
1227
1228   /**
1229    * Validate that the field requirements are met in the params.
1230    *
1231    * @param array $requiredFields
1232    * @param array $params
1233    *   An array of required fields (fieldName => label)
1234    *   - note this follows the and / or array nesting we see in permission checks
1235    *   eg.
1236    *   [
1237    *     'email' => ts('Email'),
1238    *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1239    *   ]
1240    *   Means 'email' OR 'first_name AND 'last_name'.
1241    * @param string $prefixString
1242    *
1243    * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1244    */
1245   protected function validateRequiredFields(array $requiredFields, array $params, $prefixString): void {
1246     $missingFields = [];
1247     foreach ($requiredFields as $key => $required) {
1248       if (!is_array($required)) {
1249         $importParameter = $params[$key] ?? [];
1250         if (!is_array($importParameter)) {
1251           if (!empty($importParameter)) {
1252             return;
1253           }
1254         }
1255         else {
1256           foreach ($importParameter as $locationValues) {
1257             if (!empty($locationValues[$key])) {
1258               return;
1259             }
1260           }
1261         }
1262
1263         $missingFields[$key] = $required;
1264       }
1265       else {
1266         foreach ($required as $field => $label) {
1267           if (empty($params[$field])) {
1268             $missing[$field] = $label;
1269           }
1270         }
1271         if (empty($missing)) {
1272           return;
1273         }
1274         $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1275       }
1276     }
1277     throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1278   }
1279
1280   /**
1281    * Get the field value, transformed by metadata.
1282    *
1283    * @param string $fieldName
1284    * @param string|int $importedValue
1285    *   Value as it came in from the datasource.
1286    *
1287    * @return string|array|bool|int
1288    * @throws \API_Exception
1289    */
1290   protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1291     if (empty($importedValue)) {
1292       return $importedValue;
1293     }
1294     $fieldMetadata = $this->getFieldMetadata($fieldName);
1295     if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) {
1296       $values = [];
1297       foreach (explode(',', $importedValue) as $value) {
1298         $values[] = $this->getTransformedFieldValue($fieldName, $value);
1299       }
1300       return $values;
1301     }
1302     if ($fieldName === 'url') {
1303       return CRM_Utils_Rule::url($importedValue) ? $importedValue : 'invalid_import_value';
1304     }
1305
1306     if ($fieldName === 'email') {
1307       return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value';
1308     }
1309
1310     if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) {
1311       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1312     }
1313     if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) {
1314       return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value';
1315     }
1316     if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) {
1317       $value = CRM_Utils_String::strtoboolstr($importedValue);
1318       if ($value !== FALSE) {
1319         return (bool) $value;
1320       }
1321       return 'invalid_import_value';
1322     }
1323     if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) {
1324       $value = CRM_Utils_Date::formatDate($importedValue, $this->getSubmittedValue('dateFormats'));
1325       return ($value) ?: 'invalid_import_value';
1326     }
1327     $options = $this->getFieldOptions($fieldName);
1328     if ($options !== FALSE) {
1329       if ($this->isAmbiguous($fieldName, $importedValue)) {
1330         // We can't transform it at this stage. Perhaps later we can with
1331         // other information such as country.
1332         return $importedValue;
1333       }
1334
1335       $comparisonValue = is_numeric($importedValue) ? $importedValue : mb_strtolower($importedValue);
1336       return $options[$comparisonValue] ?? 'invalid_import_value';
1337     }
1338     return $importedValue;
1339   }
1340
1341   /**
1342    * @param string $fieldName
1343    *
1344    * @return false|array
1345    *
1346    * @throws \API_Exception
1347    */
1348   protected function getFieldOptions(string $fieldName) {
1349     return $this->getFieldMetadata($fieldName, TRUE)['options'];
1350   }
1351
1352   /**
1353    * Get the metadata for the field.
1354    *
1355    * @param string $fieldName
1356    * @param bool $loadOptions
1357    * @param bool $limitToContactType
1358    *   Only show fields for the type to import (not appropriate when looking up
1359    *   related contact fields).
1360    *
1361    *
1362    * @return array
1363    * @throws \API_Exception
1364    * @throws \Civi\API\Exception\NotImplementedException
1365    */
1366   protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1367
1368     $fieldMap = $this->getOddlyMappedMetadataFields();
1369     $fieldMapName = empty($fieldMap[$fieldName]) ? $fieldName : $fieldMap[$fieldName];
1370
1371     // This whole business of only loading metadata for one type when we actually need it for all is ... dubious.
1372     if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) {
1373       if ($loadOptions || !$limitToContactType) {
1374         $this->importableFieldsMetadata[$fieldMapName] = CRM_Contact_BAO_Contact::importableFields('All')[$fieldMapName];
1375       }
1376     }
1377
1378     $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName];
1379     if ($loadOptions && !isset($fieldMetadata['options'])) {
1380       if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') {
1381         // Probably already loaded and also supports abbreviations - eg. NSW.
1382         // Supporting for core AND custom state fields is more consistent.
1383         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id');
1384         return $this->importableFieldsMetadata[$fieldMapName];
1385       }
1386       if (($fieldMetadata['data_type'] ?? '') === 'Country') {
1387         // Probably already loaded and also supports abbreviations - eg. NSW.
1388         // Supporting for core AND custom state fields is more consistent.
1389         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('country_id');
1390         return $this->importableFieldsMetadata[$fieldMapName];
1391       }
1392       $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;
1393
1394       if (!empty($fieldMetadata['custom_group_id'])) {
1395         $customField = CustomField::get(FALSE)
1396           ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
1397           ->addSelect('name', 'custom_group_id.name')
1398           ->execute()
1399           ->first();
1400         $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
1401       }
1402       $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1403         'loadOptions' => ['id', 'name', 'label', 'abbr'],
1404         'where' => [['name', '=', $optionFieldName]],
1405         'select' => ['options'],
1406       ])->first()['options'];
1407       if (is_array($options)) {
1408         // We create an array of the possible variants - notably including
1409         // name AND label as either might be used. We also lower case before checking
1410         $values = [];
1411         foreach ($options as $option) {
1412           $idKey = is_numeric($option['id']) ? $option['id'] : mb_strtolower($option['id']);
1413           $values[$idKey] = $option['id'];
1414           foreach (['name', 'label', 'abbr'] as $key) {
1415             $optionValue = mb_strtolower($option[$key] ?? '');
1416             if ($optionValue !== '') {
1417               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
1418                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
1419                   $this->ambiguousOptions[$fieldName][$optionValue] = [$values[$optionValue]];
1420                 }
1421                 $this->ambiguousOptions[$fieldName][$optionValue][] = $option['id'];
1422               }
1423               else {
1424                 $values[$optionValue] = $option['id'];
1425               }
1426             }
1427           }
1428         }
1429         $this->importableFieldsMetadata[$fieldMapName]['options'] = $values;
1430       }
1431       else {
1432         $this->importableFieldsMetadata[$fieldMapName]['options'] = $options;
1433       }
1434       return $this->importableFieldsMetadata[$fieldMapName];
1435     }
1436     return $fieldMetadata;
1437   }
1438
1439   /**
1440    * @param $customFieldID
1441    * @param $value
1442    * @param array $fieldMetaData
1443    * @param $dateType
1444    *
1445    * @return ?string
1446    */
1447   protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?string {
1448     /* validate the data against the CF type */
1449
1450     if ($value) {
1451       $dataType = $fieldMetaData['data_type'];
1452       $htmlType = $fieldMetaData['html_type'];
1453       $isSerialized = CRM_Core_BAO_CustomField::isSerialized($fieldMetaData);
1454       if ($dataType === 'Date') {
1455         $params = ['date_field' => $value];
1456         if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, 'date_field')) {
1457           return NULL;
1458         }
1459         return $fieldMetaData['label'];
1460       }
1461       elseif ($dataType === 'Boolean') {
1462         if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1463           return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1464         }
1465       }
1466       // need not check for label filed import
1467       $selectHtmlTypes = [
1468         'CheckBox',
1469         'Select',
1470         'Radio',
1471       ];
1472       if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1473         $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1474         if (!$valid) {
1475           return $fieldMetaData['label'];
1476         }
1477       }
1478
1479       // check for values for custom fields for checkboxes and multiselect
1480       if ($isSerialized && $dataType != 'ContactReference') {
1481         $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1482         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1483         foreach ($mulValues as $v1) {
1484
1485           $flag = FALSE;
1486           foreach ($customOption as $v2) {
1487             if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1488               $flag = TRUE;
1489             }
1490           }
1491
1492           if (!$flag) {
1493             return $fieldMetaData['label'];
1494           }
1495         }
1496       }
1497       elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1498         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1499         $flag = FALSE;
1500         foreach ($customOption as $v2) {
1501           if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1502             $flag = TRUE;
1503           }
1504         }
1505         if (!$flag) {
1506           return $fieldMetaData['label'];
1507         }
1508       }
1509     }
1510
1511     return NULL;
1512   }
1513
1514   /**
1515    * Get the entity for the given field.
1516    *
1517    * @param string $fieldName
1518    *
1519    * @return mixed|null
1520    * @throws \API_Exception
1521    */
1522   protected function getFieldEntity(string $fieldName) {
1523     if ($fieldName === 'do_not_import') {
1524       return NULL;
1525     }
1526     if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) {
1527       return 'Contact';
1528     }
1529     $metadata = $this->getFieldMetadata($fieldName);
1530     if (!isset($metadata['entity'])) {
1531       return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ? 'Contact' : $metadata['extends'];
1532     }
1533
1534     // Our metadata for these is fugly. Handling the fugliness during retrieval.
1535     if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1536       return 'Address';
1537     }
1538     return $metadata['entity'];
1539   }
1540
1541   /**
1542    * Validate the import file, updating the import table with results.
1543    *
1544    * @throws \API_Exception
1545    * @throws \CRM_Core_Exception
1546    */
1547   public function validate(): void {
1548     $dataSource = $this->getDataSourceObject();
1549     while ($row = $dataSource->getRow()) {
1550       try {
1551         $rowNumber = $row['_id'];
1552         $values = array_values($row);
1553         $this->validateValues($values);
1554         $this->setImportStatus($rowNumber, 'NEW', '');
1555       }
1556       catch (CRM_Core_Exception $e) {
1557         $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
1558       }
1559     }
1560   }
1561
1562   /**
1563    * Search the value for the string 'invalid_import_value'.
1564    *
1565    * If the string is found it indicates the fields was rejected
1566    * during `getTransformedValue` as not having valid data.
1567    *
1568    * @param string|array|int $value
1569    * @param string $key
1570    * @param string $prefixString
1571    *
1572    * @return array
1573    * @throws \API_Exception
1574    */
1575   protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
1576     $errors = [];
1577     if ($value === 'invalid_import_value') {
1578       $metadata = $this->getFieldMetadata($key);
1579       $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1580     }
1581     elseif (is_array($value)) {
1582       foreach ($value as $innerKey => $innerValue) {
1583         $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1584         if (!empty($result)) {
1585           $errors = array_merge($result, $errors);
1586         }
1587       }
1588     }
1589     return array_filter($errors);
1590   }
1591
1592   /**
1593    * Get the available countries.
1594    *
1595    * If the site is not configured with a restriction then all countries are valid
1596    * but otherwise only a select array are.
1597    *
1598    * @return array|false
1599    *   FALSE indicates no restrictions.
1600    */
1601   protected function getAvailableCountries() {
1602     if ($this->availableCountries === NULL) {
1603       $availableCountries = Civi::settings()->get('countryLimit');
1604       $this->availableCountries = !empty($availableCountries) ? array_fill_keys($availableCountries, TRUE) : FALSE;
1605     }
1606     return $this->availableCountries;
1607   }
1608
1609   /**
1610    * Get the metadata field for which importable fields does not key the actual field name.
1611    *
1612    * @return string[]
1613    */
1614   protected function getOddlyMappedMetadataFields(): array {
1615     return [
1616       'country_id' => 'country',
1617       'state_province_id' => 'state_province',
1618       'county_id' => 'county',
1619       'email_greeting_id' => 'email_greeting',
1620       'postal_greeting_id' => 'postal_greeting',
1621       'addressee_id' => 'addressee',
1622     ];
1623   }
1624
1625   /**
1626    * Get the default country for the site.
1627    *
1628    * @return int
1629    */
1630   protected function getSiteDefaultCountry(): int {
1631     if (!isset($this->siteDefaultCountry)) {
1632       $this->siteDefaultCountry = (int) Civi::settings()->get('defaultContactCountry');
1633     }
1634     return $this->siteDefaultCountry;
1635   }
1636
1637   /**
1638    * Is the option ambiguous.
1639    *
1640    * @param string $fieldName
1641    * @param string $importedValue
1642    */
1643   protected function isAmbiguous(string $fieldName, $importedValue): bool {
1644     return !empty($this->ambiguousOptions[$fieldName][mb_strtolower($importedValue)]);
1645   }
1646
1647   /**
1648    * Get the civicrm_mapping_field appropriate layout for the mapper input.
1649    *
1650    * For simple parsers (not contribution or contact) the input looks like
1651    * ['first_name', 'custom_32']
1652    * and it is converted to
1653    *
1654    *  ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5],
1655    *
1656    * @param array $fieldMapping
1657    * @param int $mappingID
1658    * @param int $columnNumber
1659    *
1660    * @return array
1661    */
1662   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
1663     return [
1664       'name' => $fieldMapping[0],
1665       'mapping_id' => $mappingID,
1666       'column_number' => $columnNumber,
1667     ];
1668   }
1669
1670   /**
1671    * @param array $mappedField
1672    *   Field detail as would be saved in field_mapping table
1673    *   or as returned from getMappingFieldFromMapperInput
1674    *
1675    * @return string
1676    * @throws \API_Exception
1677    */
1678   public function getMappedFieldLabel(array $mappedField): string {
1679     $this->setFieldMetadata();
1680     return $this->getFieldMetadata($mappedField['name'])['title'];
1681   }
1682
1683   /**
1684    * Get the row from the csv mapped to our parameters.
1685    *
1686    * @param array $values
1687    *
1688    * @return array
1689    * @throws \API_Exception
1690    */
1691   public function getMappedRow(array $values): array {
1692     $params = [];
1693     foreach ($this->getFieldMappings() as $i => $mappedField) {
1694       if ($mappedField['name'] === 'do_not_import') {
1695         continue;
1696       }
1697       if ($mappedField['name']) {
1698         $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
1699       }
1700     }
1701     return $params;
1702   }
1703
1704   /**
1705    * Get the field mappings for the import.
1706    *
1707    * This is the same format as saved in civicrm_mapping_field except
1708    * that location_type_id = 'Primary' rather than empty where relevant.
1709    * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
1710    *
1711    * @return array
1712    * @throws \API_Exception
1713    */
1714   protected function getFieldMappings(): array {
1715     $mappedFields = [];
1716     $mapper = $this->getSubmittedValue('mapper');
1717     foreach ($mapper as $i => $mapperRow) {
1718       $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
1719       // Just for clarity since 0 is a pseudo-value
1720       unset($mappedField['mapping_id']);
1721       $mappedFields[] = $mappedField;
1722     }
1723     return $mappedFields;
1724   }
1725
1726   /**
1727    * Run import.
1728    *
1729    * @param \CRM_Queue_TaskContext $taskContext
1730    *
1731    * @param int $userJobID
1732    * @param int $limit
1733    *
1734    * @return bool
1735    * @throws \API_Exception
1736    * @throws \CRM_Core_Exception
1737    */
1738   public static function runImport($taskContext, $userJobID, $limit) {
1739     $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('type_id')->execute()->first();
1740     $parserClass = NULL;
1741     foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
1742       if ($userJob['type_id'] === $userJobType['id']) {
1743         $parserClass = $userJobType['class'];
1744       }
1745     }
1746     /* @var \CRM_Import_Parser $parser */
1747     $parser = new $parserClass();
1748     $parser->setUserJobID($userJobID);
1749     // Not sure if we still need to init....
1750     $parser->init();
1751     $dataSource = $parser->getDataSourceObject();
1752     $dataSource->setStatuses(['new']);
1753     $dataSource->setLimit($limit);
1754
1755     while ($row = $dataSource->getRow()) {
1756       $values = array_values($row);
1757       $parser->import($values);
1758     }
1759     $parser->doPostImportActions();
1760     return TRUE;
1761   }
1762
1763   /**
1764    * Check if an error in custom data.
1765    *
1766    * @deprecated all of this is duplicated if getTransformedValue is used.
1767    *
1768    * @param array $params
1769    * @param string $errorMessage
1770    *   A string containing all the error-fields.
1771    *
1772    * @param null $csType
1773    */
1774   public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) {
1775     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1776     $errors = [];
1777
1778     if (!empty($params['contact_sub_type'])) {
1779       $csType = $params['contact_sub_type'] ?? NULL;
1780     }
1781
1782     if (empty($params['contact_type'])) {
1783       $params['contact_type'] = 'Individual';
1784     }
1785
1786     // get array of subtypes - CRM-18708
1787     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1788       $csType = $this->getSubtypes($params['contact_type']);
1789     }
1790
1791     if (is_array($csType)) {
1792       // fetch custom fields for every subtype and add it to $customFields array
1793       // CRM-18708
1794       $customFields = [];
1795       foreach ($csType as $cType) {
1796         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1797       }
1798     }
1799     else {
1800       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1801     }
1802
1803     foreach ($params as $key => $value) {
1804       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1805         //For address custom fields, we do get actual custom field value as an inner array of
1806         //values so need to modify
1807         if (!array_key_exists($customFieldID, $customFields)) {
1808           return ts('field ID');
1809         }
1810         /* check if it's a valid custom field id */
1811         $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType);
1812       }
1813     }
1814     if ($errors) {
1815       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', array_filter($errors));
1816     }
1817   }
1818
1819   /**
1820    * get subtypes given the contact type
1821    *
1822    * @param string $contactType
1823    * @return array $subTypes
1824    */
1825   protected function getSubtypes($contactType) {
1826     $subTypes = [];
1827     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
1828
1829     if (count($types) > 0) {
1830       foreach ($types as $type) {
1831         $subTypes[] = $type['name'];
1832       }
1833     }
1834     return $subTypes;
1835   }
1836
1837   /**
1838    * Update the status of the import row to reflect the processing outcome.
1839    *
1840    * @param int $id
1841    * @param string $status
1842    * @param string $message
1843    * @param int|null $entityID
1844    *   Optional created entity ID
1845    *
1846    * @noinspection PhpDocMissingThrowsInspection
1847    * @noinspection PhpUnhandledExceptionInspection
1848    */
1849   protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL): void {
1850     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID);
1851   }
1852
1853   /**
1854    * Convert any given date string to default date array.
1855    *
1856    * @param array $params
1857    *   Has given date-format.
1858    * @param array $formatted
1859    *   Store formatted date in this array.
1860    * @param int $dateType
1861    *   Type of date.
1862    * @param string $dateParam
1863    *   Index of params.
1864    */
1865   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
1866     //fix for CRM-2687
1867     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
1868     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
1869   }
1870
1871 }