CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Campaign;
  13 use Civi\Api4\CustomField;
  14 use Civi\Api4\Event;
  15 use Civi\Api4\UserJob;
  16
  17 /**
  18  *
  19  * @package CRM
  20  * @copyright CiviCRM LLC https://civicrm.org/licensing
  21  */
  22 abstract class CRM_Import_Parser {
  23   /**
  24    * Settings
  25    */
  26   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  27
  28   /**
  29    * Return codes
  30    */
  31   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  32
  33   /**
  34    * Parser modes
  35    */
  36   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  37
  38   /**
  39    * Codes for duplicate record handling
  40    */
  41   const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  42
  43   /**
  44    * Contact types
  45    */
  46   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  47
  48   /**
  49    * User job id.
  50    *
  51    * This is the primary key of the civicrm_user_job table which is used to
  52    * track the import.
  53    *
  54    * @var int
  55    */
  56   protected $userJobID;
  57
  58   /**
  59    * The user job in use.
  60    *
  61    * @var array
  62    */
  63   protected $userJob;
  64
  65   /**
  66    * Potentially ambiguous options.
  67    *
  68    * For example 'UT' is a state in more than one country.
  69    *
  70    * @var array
  71    */
  72   protected $ambiguousOptions = [];
  73
  74   /**
  75    * States to country mapping.
  76    *
  77    * @var array
  78    */
  79   protected $statesByCountry = [];
  80
  81   /**
  82    * @return int|null
  83    */
  84   public function getUserJobID(): ?int {
  85     return $this->userJobID;
  86   }
  87
  88   /**
  89    * Ids of contacts created this iteration.
  90    *
  91    * @var array
  92    */
  93   protected $createdContacts = [];
  94
  95   /**
  96    * Set user job ID.
  97    *
  98    * @param int $userJobID
  99    *
 100    * @return self
 101    */
 102   public function setUserJobID(int $userJobID): self {
 103     $this->userJobID = $userJobID;
 104     return $this;
 105   }
 106
 107   /**
 108    * Countries that the site is restricted to
 109    *
 110    * @var array|false
 111    */
 112   private $availableCountries;
 113
 114   /**
 115    *
 116    * @return array
 117    */
 118   public function getTrackingFields(): array {
 119     return [];
 120   }
 121
 122   /**
 123    * Get User Job.
 124    *
 125    * API call to retrieve the userJob row.
 126    *
 127    * @return array
 128    *
 129    * @throws \API_Exception
 130    */
 131   protected function getUserJob(): array {
 132     if (empty($this->userJob)) {
 133       $this->userJob = UserJob::get()
 134         ->addWhere('id', '=', $this->getUserJobID())
 135         ->execute()
 136         ->first();
 137     }
 138     return $this->userJob;
 139   }
 140
 141   /**
 142    * Get the relevant datasource object.
 143    *
 144    * @return \CRM_Import_DataSource|null
 145    *
 146    * @throws \API_Exception
 147    */
 148   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 149     $className = $this->getSubmittedValue('dataSource');
 150     if ($className) {
 151       /* @var CRM_Import_DataSource $dataSource */
 152       return new $className($this->getUserJobID());
 153     }
 154     return NULL;
 155   }
 156
 157   /**
 158    * Get the submitted value, as stored on the user job.
 159    *
 160    * @param string $fieldName
 161    *
 162    * @return mixed
 163    *
 164    * @noinspection PhpDocMissingThrowsInspection
 165    * @noinspection PhpUnhandledExceptionInspection
 166    */
 167   protected function getSubmittedValue(string $fieldName) {
 168     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 169   }
 170
 171   /**
 172    * Has the import completed.
 173    *
 174    * @return bool
 175    *
 176    * @throws \API_Exception
 177    * @throws \CRM_Core_Exception
 178    */
 179   public function isComplete() :bool {
 180     return $this->getDataSourceObject()->isCompleted();
 181   }
 182
 183   /**
 184    * Get configured contact type.
 185    *
 186    * @return string
 187    */
 188   protected function getContactType(): string {
 189     if (!$this->_contactType) {
 190       $contactTypeMapping = [
 191         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 192         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 193         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 194       ];
 195       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 196     }
 197     return $this->_contactType;
 198   }
 199
 200   /**
 201    * Get configured contact type.
 202    *
 203    * @return string|null
 204    */
 205   public function getContactSubType(): ?string {
 206     if (!$this->_contactSubType) {
 207       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 208     }
 209     return $this->_contactSubType;
 210   }
 211
 212   /**
 213    * Total number of non empty lines
 214    * @var int
 215    */
 216   protected $_totalCount;
 217
 218   /**
 219    * Running total number of valid lines
 220    * @var int
 221    */
 222   protected $_validCount;
 223
 224   /**
 225    * Running total number of invalid rows
 226    * @var int
 227    */
 228   protected $_invalidRowCount;
 229
 230   /**
 231    * Maximum number of non-empty/comment lines to process
 232    *
 233    * @var int
 234    */
 235   protected $_maxLinesToProcess;
 236
 237   /**
 238    * Array of error lines, bounded by MAX_ERROR
 239    * @var array
 240    */
 241   protected $_errors;
 242
 243   /**
 244    * Total number of duplicate (from database) lines
 245    * @var int
 246    */
 247   protected $_duplicateCount;
 248
 249   /**
 250    * Array of duplicate lines
 251    * @var array
 252    */
 253   protected $_duplicates;
 254
 255   /**
 256    * Maximum number of warnings to store
 257    * @var int
 258    */
 259   protected $_maxWarningCount = self::MAX_WARNINGS;
 260
 261   /**
 262    * Array of warning lines, bounded by MAX_WARNING
 263    * @var array
 264    */
 265   protected $_warnings;
 266
 267   /**
 268    * Array of all the fields that could potentially be part
 269    * of this import process
 270    * @var array
 271    */
 272   protected $_fields;
 273
 274   /**
 275    * Metadata for all available fields, keyed by unique name.
 276    *
 277    * This is intended to supercede $_fields which uses a special sauce format which
 278    * importableFieldsMetadata uses the standard getfields type format.
 279    *
 280    * @var array
 281    */
 282   protected $importableFieldsMetadata = [];
 283
 284   /**
 285    * Get metadata for all importable fields in std getfields style format.
 286    *
 287    * @return array
 288    */
 289   public function getImportableFieldsMetadata(): array {
 290     return $this->importableFieldsMetadata;
 291   }
 292
 293   /**
 294    * Set metadata for all importable fields in std getfields style format.
 295    *
 296    * @param array $importableFieldsMetadata
 297    */
 298   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 299     $this->importableFieldsMetadata = $importableFieldsMetadata;
 300   }
 301
 302   /**
 303    * Gets the fields available for importing in a key-name, title format.
 304    *
 305    * @return array
 306    *   eg. ['first_name' => 'First Name'.....]
 307    *
 308    * @throws \API_Exception
 309    *
 310    * @todo - we are constructing the metadata before we
 311    * have set the contact type so we re-do it here.
 312    *
 313    * Once we have cleaned up the way the mapper is handled
 314    * we can ditch all the existing _construct parameters in favour
 315    * of just the userJobID - there are current open PRs towards this end.
 316    */
 317   public function getAvailableFields(): array {
 318     $this->setFieldMetadata();
 319     $return = [];
 320     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 321       if ($name === 'id' && $this->isSkipDuplicates()) {
 322         // Duplicates are being skipped so id matching is not availble.
 323         continue;
 324       }
 325       $return[$name] = $field['html']['label'] ?? $field['title'];
 326     }
 327     return $return;
 328   }
 329
 330   /**
 331    * Did the user specify duplicates should be skipped and not imported.
 332    *
 333    * @return bool
 334    *
 335    * @throws \API_Exception
 336    */
 337   protected function isSkipDuplicates(): bool {
 338     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 339   }
 340
 341   /**
 342    * Is this a case where the user has opted to update existing contacts.
 343    *
 344    * @return bool
 345    */
 346   protected function isUpdateExisting(): bool {
 347     return in_array((int) $this->getSubmittedValue('onDuplicate'), [
 348       CRM_Import_Parser::DUPLICATE_UPDATE,
 349       CRM_Import_Parser::DUPLICATE_FILL,
 350     ], TRUE);
 351   }
 352
 353   /**
 354    * Did the user specify duplicates checking should be skipped, resulting in possible duplicate contacts.
 355    *
 356    * Note we still need to check for external_identifier as it will hard-fail
 357    * if we duplicate.
 358    *
 359    * @return bool
 360    */
 361   protected function isIgnoreDuplicates(): bool {
 362     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_NOCHECK;
 363   }
 364
 365   /**
 366    * Did the user specify duplicates should be filled with missing data.
 367    *
 368    * @return bool
 369    */
 370   protected function isFillDuplicates(): bool {
 371     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_FILL;
 372   }
 373
 374   /**
 375    * Array of the fields that are actually part of the import process
 376    * the position in the array also dictates their position in the import
 377    * file
 378    * @var array
 379    */
 380   protected $_activeFields = [];
 381
 382   /**
 383    * Cache the count of active fields
 384    *
 385    * @var int
 386    */
 387   protected $_activeFieldCount;
 388
 389   /**
 390    * Cache of preview rows
 391    *
 392    * @var array
 393    */
 394   protected $_rows;
 395
 396   /**
 397    * Filename of error data
 398    *
 399    * @var string
 400    */
 401   protected $_errorFileName;
 402
 403   /**
 404    * Filename of duplicate data
 405    *
 406    * @var string
 407    */
 408   protected $_duplicateFileName;
 409
 410   /**
 411    * Contact type
 412    *
 413    * @var string
 414    */
 415   public $_contactType;
 416
 417   /**
 418    * @param string $contactType
 419    *
 420    * @return CRM_Import_Parser
 421    */
 422   public function setContactType(string $contactType): CRM_Import_Parser {
 423     $this->_contactType = $contactType;
 424     return $this;
 425   }
 426
 427   /**
 428    * Contact sub-type
 429    *
 430    * @var int|null
 431    */
 432   public $_contactSubType;
 433
 434   /**
 435    * @param int|null $contactSubType
 436    *
 437    * @return self
 438    */
 439   public function setContactSubType(?int $contactSubType): self {
 440     $this->_contactSubType = $contactSubType;
 441     return $this;
 442   }
 443
 444   /**
 445    * Class constructor.
 446    */
 447   public function __construct() {
 448     $this->_maxLinesToProcess = 0;
 449   }
 450
 451   /**
 452    * Set and validate field values.
 453    *
 454    * @param array $elements
 455    *   array.
 456    */
 457   public function setActiveFieldValues($elements): void {
 458     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 459     for ($i = 0; $i < $maxCount; $i++) {
 460       $this->_activeFields[$i]->setValue($elements[$i]);
 461     }
 462
 463     // reset all the values that we did not have an equivalent import element
 464     for (; $i < $this->_activeFieldCount; $i++) {
 465       $this->_activeFields[$i]->resetValue();
 466     }
 467   }
 468
 469   /**
 470    * Format the field values for input to the api.
 471    *
 472    * @return array
 473    *   (reference) associative array of name/value pairs
 474    */
 475   public function &getActiveFieldParams() {
 476     $params = [];
 477     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 478       if (isset($this->_activeFields[$i]->_value)
 479         && !isset($params[$this->_activeFields[$i]->_name])
 480         && !isset($this->_activeFields[$i]->_related)
 481       ) {
 482
 483         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 484       }
 485     }
 486     return $params;
 487   }
 488
 489   /**
 490    * Add progress bar to the import process. Calculates time remaining, status etc.
 491    *
 492    * @param $statusID
 493    *   status id of the import process saved in $config->uploadDir.
 494    * @param bool $startImport
 495    *   True when progress bar is to be initiated.
 496    * @param $startTimestamp
 497    *   Initial timestamp when the import was started.
 498    * @param $prevTimestamp
 499    *   Previous timestamp when this function was last called.
 500    * @param $totalRowCount
 501    *   Total number of rows in the import file.
 502    *
 503    * @return NULL|$currTimestamp
 504    */
 505   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 506     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 507
 508     if ($startImport) {
 509       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 510       //do not force the browser to display the save dialog, CRM-7640
 511       $contents = json_encode([0, $status]);
 512       file_put_contents($statusFile, $contents);
 513     }
 514     else {
 515       $rowCount = $this->_rowCount ?? $this->_lineCount;
 516       $currTimestamp = time();
 517       $time = ($currTimestamp - $prevTimestamp);
 518       $recordsLeft = $totalRowCount - $rowCount;
 519       if ($recordsLeft < 0) {
 520         $recordsLeft = 0;
 521       }
 522       $estimatedTime = ($recordsLeft / 50) * $time;
 523       $estMinutes = floor($estimatedTime / 60);
 524       $timeFormatted = '';
 525       if ($estMinutes > 1) {
 526         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 527         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 528       }
 529       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 530       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 531       $statusMsg = ts('%1 of %2 records - %3 remaining',
 532         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 533       );
 534       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 535       $contents = json_encode([$processedPercent, $status]);
 536
 537       file_put_contents($statusFile, $contents);
 538       return $currTimestamp;
 539     }
 540   }
 541
 542   /**
 543    * @return array
 544    */
 545   public function getSelectValues(): array {
 546     $values = [];
 547     foreach ($this->_fields as $name => $field) {
 548       $values[$name] = $field->_title;
 549     }
 550     return $values;
 551   }
 552
 553   /**
 554    * @return array
 555    */
 556   public function getSelectTypes() {
 557     $values = [];
 558     // This is only called from the MapField form in isolation now,
 559     // so we need to set the metadata.
 560     $this->init();
 561     foreach ($this->_fields as $name => $field) {
 562       if (isset($field->_hasLocationType)) {
 563         $values[$name] = $field->_hasLocationType;
 564       }
 565     }
 566     return $values;
 567   }
 568
 569   /**
 570    * @return array
 571    */
 572   public function getHeaderPatterns(): array {
 573     $values = [];
 574     foreach ($this->importableFieldsMetadata as $name => $field) {
 575       if (isset($field['headerPattern'])) {
 576         $values[$name] = $field['headerPattern'] ?: '//';
 577       }
 578     }
 579     return $values;
 580   }
 581
 582   /**
 583    * @return array
 584    */
 585   public function getDataPatterns():array {
 586     $values = [];
 587     foreach ($this->_fields as $name => $field) {
 588       $values[$name] = $field->_dataPattern;
 589     }
 590     return $values;
 591   }
 592
 593   /**
 594    * Remove single-quote enclosures from a value array (row).
 595    *
 596    * @param array $values
 597    * @param string $enclosure
 598    *
 599    * @return void
 600    */
 601   public static function encloseScrub(&$values, $enclosure = "'") {
 602     if (empty($values)) {
 603       return;
 604     }
 605
 606     foreach ($values as $k => $v) {
 607       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 608     }
 609   }
 610
 611   /**
 612    * Setter function.
 613    *
 614    * @param int $max
 615    *
 616    * @return void
 617    */
 618   public function setMaxLinesToProcess($max) {
 619     $this->_maxLinesToProcess = $max;
 620   }
 621
 622   /**
 623    * Validate that we have the required fields to create the contact or find it to update.
 624    *
 625    * Note that the users duplicate selection affects this as follows
 626    * - if they did not select an update variant then the id field is not
 627    *   permitted in the mapping - so we can assume the presence of id means
 628    *   we should use it
 629    * - the external_identifier field is valid in place of the other fields
 630    *   when they have chosen update or fill - in this case we are only looking
 631    *   to update an existing contact.
 632    *
 633    * @param string $contactType
 634    * @param array $params
 635    * @param bool $isPermitExistingMatchFields
 636    *   True if the it is enough to have fields which will enable us to find
 637    *   an existing contact (eg. external_identifier).
 638    * @param string $prefixString
 639    *   String to include in the exception (e.g '(Child of)' if we are validating
 640    *   a related contact.
 641    *
 642    * @return void
 643    * @throws \CRM_Core_Exception
 644    */
 645   protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void {
 646     if (!empty($params['id'])) {
 647       return;
 648     }
 649     $requiredFields = [
 650       'Individual' => [
 651         'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
 652         'email' => ts('Email Address'),
 653       ],
 654       'Organization' => ['organization_name' => ts('Organization Name')],
 655       'Household' => ['household_name' => ts('Household Name')],
 656     ][$contactType];
 657     if ($isPermitExistingMatchFields) {
 658       $requiredFields['external_identifier'] = ts('External Identifier');
 659       // Historically just an email has been accepted as it is 'usually good enough'
 660       // for a dedupe rule look up - but really this is a stand in for
 661       // whatever is needed to find an existing matching contact using the
 662       // specified dedupe rule (or the default Unsupervised if not specified).
 663       $requiredFields['email'] = ts('Email Address');
 664     }
 665     $this->validateRequiredFields($requiredFields, $params, $prefixString);
 666   }
 667
 668   protected function doPostImportActions() {
 669     $userJob = $this->getUserJob();
 670     $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
 671     $actions = $userJob['metadata']['post_actions'] ?? [];
 672     if (!empty($actions['group'])) {
 673       $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']);
 674       foreach ($actions['group'] as $groupID) {
 675         $summaryInfo['groups'][$groupID]['added'] += $groupAdditions[$groupID]['added'];
 676         $summaryInfo['groups'][$groupID]['notAdded'] += $groupAdditions[$groupID]['notAdded'];
 677       }
 678     }
 679     if (!empty($actions['tag'])) {
 680       $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts, $actions['tag']);
 681       foreach ($actions['tag'] as $tagID) {
 682         $summaryInfo['tags'][$tagID]['added'] += $tagAdditions[$tagID]['added'];
 683         $summaryInfo['tags'][$tagID]['notAdded'] += $tagAdditions[$tagID]['notAdded'];
 684       }
 685     }
 686
 687     $this->userJob['metadata']['summary_info'] = $summaryInfo;
 688     UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute();
 689   }
 690
 691   public function queue() {
 692     $dataSource = $this->getDataSourceObject();
 693     $totalRowCount = $totalRows = $dataSource->getRowCount(['new']);
 694     $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']);
 695     $offset = 0;
 696     $batchSize = 5;
 697     while ($totalRows > 0) {
 698       if ($totalRows < $batchSize) {
 699         $batchSize = $totalRows;
 700       }
 701       $task = new CRM_Queue_Task(
 702         [get_class($this), 'runImport'],
 703         ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize],
 704         ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount])
 705       );
 706       $queue->createItem($task);
 707       $totalRows -= $batchSize;
 708       $offset += $batchSize;
 709     }
 710
 711   }
 712
 713   /**
 714    * Add imported contacts to groups.
 715    *
 716    * @param array $contactIDs
 717    * @param array $groups
 718    *
 719    * @return array
 720    */
 721   private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array {
 722     $groupAdditions = [];
 723     foreach ($groups as $groupID) {
 724       // @todo - this function has been in use historically but it does not seem
 725       // to add much efficiency of get + create api calls
 726       // and it doesn't give enough control over cache flushing for smaller batches.
 727       // Note that the import updates a lot of enities & checking & updating the group
 728       // shouldn't add much performance wise. However, cache flushing will
 729       $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIDs, $groupID);
 730       $groupAdditions[$groupID] = [
 731         'added' => (int) $addCount[1],
 732         'notAdded' => (int) $addCount[2],
 733       ];
 734     }
 735     return $groupAdditions;
 736   }
 737
 738   /**
 739    * Tag imported contacts.
 740    *
 741    * @param array $contactIDs
 742    * @param array $tags
 743    *
 744    * @return array
 745    */
 746   private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) {
 747     $tagAdditions = [];
 748     foreach ($tags as $tagID) {
 749       // @todo - this function has been in use historically but it does not seem
 750       // to add much efficiency of get + create api calls
 751       // and it doesn't give enough control over cache flushing for smaller batches.
 752       // Note that the import updates a lot of enities & checking & updating the group
 753       // shouldn't add much performance wise. However, cache flushing will
 754       $outcome = CRM_Core_BAO_EntityTag::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE);
 755       $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]];
 756     }
 757     return $tagAdditions;
 758   }
 759
 760   /**
 761    * Determines the file extension based on error code.
 762    *
 763    * @var int $type error code constant
 764    * @return string
 765    */
 766   public static function errorFileName($type) {
 767     $fileName = NULL;
 768     if (empty($type)) {
 769       return $fileName;
 770     }
 771
 772     $config = CRM_Core_Config::singleton();
 773     $fileName = $config->uploadDir . "sqlImport";
 774     switch ($type) {
 775       case self::ERROR:
 776         $fileName .= '.errors';
 777         break;
 778
 779       case self::DUPLICATE:
 780         $fileName .= '.duplicates';
 781         break;
 782
 783       case self::NO_MATCH:
 784         $fileName .= '.mismatch';
 785         break;
 786
 787       case self::UNPARSED_ADDRESS_WARNING:
 788         $fileName .= '.unparsedAddress';
 789         break;
 790     }
 791
 792     return $fileName;
 793   }
 794
 795   /**
 796    * Determines the file name based on error code.
 797    *
 798    * @var $type error code constant
 799    * @return string
 800    */
 801   public static function saveFileName($type) {
 802     $fileName = NULL;
 803     if (empty($type)) {
 804       return $fileName;
 805     }
 806     switch ($type) {
 807       case self::ERROR:
 808         $fileName = 'Import_Errors.csv';
 809         break;
 810
 811       case self::DUPLICATE:
 812         $fileName = 'Import_Duplicates.csv';
 813         break;
 814
 815       case self::NO_MATCH:
 816         $fileName = 'Import_Mismatch.csv';
 817         break;
 818
 819       case self::UNPARSED_ADDRESS_WARNING:
 820         $fileName = 'Import_Unparsed_Address.csv';
 821         break;
 822     }
 823
 824     return $fileName;
 825   }
 826
 827   /**
 828    * Check if contact is a duplicate .
 829    *
 830    * @param array $formatValues
 831    *
 832    * @return array
 833    */
 834   protected function checkContactDuplicate(&$formatValues) {
 835     //retrieve contact id using contact dedupe rule
 836     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->getContactType();
 837     $formatValues['version'] = 3;
 838     require_once 'CRM/Utils/DeprecatedUtils.php';
 839     $params = $formatValues;
 840     static $cIndieFields = NULL;
 841     static $defaultLocationId = NULL;
 842
 843     $contactType = $params['contact_type'];
 844     if ($cIndieFields == NULL) {
 845       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 846       $cIndieFields = $cTempIndieFields;
 847
 848       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 849
 850       // set the value to default location id else set to 1
 851       if (!$defaultLocationId = (int) $defaultLocation->id) {
 852         $defaultLocationId = 1;
 853       }
 854     }
 855
 856     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 857
 858     $contactFormatted = [];
 859     foreach ($params as $key => $field) {
 860       if ($field == NULL || $field === '') {
 861         continue;
 862       }
 863       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 864       // instead of soft credit contact.
 865       if (is_array($field) && $key !== "soft_credit") {
 866         foreach ($field as $value) {
 867           $break = FALSE;
 868           if (is_array($value)) {
 869             foreach ($value as $name => $testForEmpty) {
 870               if ($name !== 'phone_type' &&
 871                 ($testForEmpty === '' || $testForEmpty == NULL)
 872               ) {
 873                 $break = TRUE;
 874                 break;
 875               }
 876             }
 877           }
 878           else {
 879             $break = TRUE;
 880           }
 881           if (!$break) {
 882             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 883           }
 884         }
 885         continue;
 886       }
 887
 888       $value = [$key => $field];
 889
 890       // check if location related field, then we need to add primary location type
 891       if (in_array($key, $locationFields)) {
 892         $value['location_type_id'] = $defaultLocationId;
 893       }
 894       elseif (array_key_exists($key, $cIndieFields)) {
 895         $value['contact_type'] = $contactType;
 896       }
 897
 898       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 899     }
 900
 901     $contactFormatted['contact_type'] = $contactType;
 902
 903     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 904   }
 905
 906   /**
 907    * This function adds the contact variable in $values to the
 908    * parameter list $params.  For most cases, $values should have length 1.  If
 909    * the variable being added is a child of Location, a location_type_id must
 910    * also be included.  If it is a child of phone, a phone_type must be included.
 911    *
 912    * @param array $values
 913    *   The variable(s) to be added.
 914    * @param array $params
 915    *   The structured parameter list.
 916    *
 917    * @return bool|CRM_Utils_Error
 918    */
 919   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 920     // @todo - like most functions in import ... most of this is cruft....
 921     // Crawl through the possible classes:
 922     // Contact
 923     //      Individual
 924     //      Household
 925     //      Organization
 926     //          Location
 927     //              Address
 928     //              Email
 929     //              Phone
 930     //              IM
 931     //      Note
 932     //      Custom
 933
 934     // Cache the various object fields
 935     static $fields = NULL;
 936
 937     if ($fields == NULL) {
 938       $fields = [];
 939     }
 940
 941     // first add core contact values since for other Civi modules they are not added
 942     require_once 'CRM/Contact/BAO/Contact.php';
 943     $contactFields = CRM_Contact_DAO_Contact::fields();
 944     _civicrm_api3_store_values($contactFields, $values, $params);
 945
 946     if (isset($values['contact_type'])) {
 947       // we're an individual/household/org property
 948
 949       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 950
 951       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 952       return TRUE;
 953     }
 954
 955     if (isset($values['individual_prefix'])) {
 956       if (!empty($params['prefix_id'])) {
 957         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 958         $params['prefix'] = $prefixes[$params['prefix_id']];
 959       }
 960       else {
 961         $params['prefix'] = $values['individual_prefix'];
 962       }
 963       return TRUE;
 964     }
 965
 966     if (isset($values['individual_suffix'])) {
 967       if (!empty($params['suffix_id'])) {
 968         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 969         $params['suffix'] = $suffixes[$params['suffix_id']];
 970       }
 971       else {
 972         $params['suffix'] = $values['individual_suffix'];
 973       }
 974       return TRUE;
 975     }
 976
 977     if (isset($values['gender'])) {
 978       if (!empty($params['gender_id'])) {
 979         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 980         $params['gender'] = $genders[$params['gender_id']];
 981       }
 982       else {
 983         $params['gender'] = $values['gender'];
 984       }
 985       return TRUE;
 986     }
 987
 988     // format the website params.
 989     if (!empty($values['url'])) {
 990       static $websiteFields;
 991       if (!is_array($websiteFields)) {
 992         require_once 'CRM/Core/DAO/Website.php';
 993         $websiteFields = CRM_Core_DAO_Website::fields();
 994       }
 995       if (!array_key_exists('website', $params) ||
 996         !is_array($params['website'])
 997       ) {
 998         $params['website'] = [];
 999       }
1000
1001       $websiteCount = count($params['website']);
1002       _civicrm_api3_store_values($websiteFields, $values,
1003         $params['website'][++$websiteCount]
1004       );
1005
1006       return TRUE;
1007     }
1008
1009     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1010     if (!empty($values['location_type_id'])) {
1011       static $fields = NULL;
1012       if ($fields == NULL) {
1013         $fields = [];
1014       }
1015
1016       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
1017         $name = strtolower($block);
1018         if (!array_key_exists($name, $values)) {
1019           continue;
1020         }
1021
1022         if ($name === 'phone_ext') {
1023           $block = 'Phone';
1024         }
1025
1026         // block present in value array.
1027         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
1028           $params[$name] = [];
1029         }
1030
1031         if (!array_key_exists($block, $fields)) {
1032           $className = "CRM_Core_DAO_$block";
1033           $fields[$block] =& $className::fields();
1034         }
1035
1036         $blockCnt = count($params[$name]);
1037
1038         // copy value to dao field name.
1039         if ($name == 'im') {
1040           $values['name'] = $values[$name];
1041         }
1042
1043         _civicrm_api3_store_values($fields[$block], $values,
1044           $params[$name][++$blockCnt]
1045         );
1046
1047         if (empty($params['id']) && ($blockCnt == 1)) {
1048           $params[$name][$blockCnt]['is_primary'] = TRUE;
1049         }
1050
1051         // we only process single block at a time.
1052         return TRUE;
1053       }
1054
1055       // handle address fields.
1056       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1057         $params['address'] = [];
1058       }
1059
1060       $addressCnt = 1;
1061       foreach ($params['address'] as $cnt => $addressBlock) {
1062         if (CRM_Utils_Array::value('location_type_id', $values) ==
1063           CRM_Utils_Array::value('location_type_id', $addressBlock)
1064         ) {
1065           $addressCnt = $cnt;
1066           break;
1067         }
1068         $addressCnt++;
1069       }
1070
1071       if (!array_key_exists('Address', $fields)) {
1072         $fields['Address'] = CRM_Core_DAO_Address::fields();
1073       }
1074
1075       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1076       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1077       // the address in CRM_Core_BAO_Address::create method
1078       if (!empty($values['location_type_id'])) {
1079         static $customFields = [];
1080         if (empty($customFields)) {
1081           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1082         }
1083         // make a copy of values, as we going to make changes
1084         $newValues = $values;
1085         foreach ($values as $key => $val) {
1086           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1087           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1088             // mark an entry in fields array since we want the value of custom field to be copied
1089             $fields['Address'][$key] = NULL;
1090
1091             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1092             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1093               $mulValues = explode(',', $val);
1094               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1095               $newValues[$key] = [];
1096               foreach ($mulValues as $v1) {
1097                 foreach ($customOption as $v2) {
1098                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1099                     (strtolower($v2['value']) == strtolower(trim($v1)))
1100                   ) {
1101                     if ($htmlType == 'CheckBox') {
1102                       $newValues[$key][$v2['value']] = 1;
1103                     }
1104                     else {
1105                       $newValues[$key][] = $v2['value'];
1106                     }
1107                   }
1108                 }
1109               }
1110             }
1111           }
1112         }
1113         // consider new values
1114         $values = $newValues;
1115       }
1116
1117       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1118
1119       $addressFields = [
1120         'county',
1121         'country',
1122         'state_province',
1123         'supplemental_address_1',
1124         'supplemental_address_2',
1125         'supplemental_address_3',
1126         'StateProvince.name',
1127       ];
1128
1129       foreach ($addressFields as $field) {
1130         if (array_key_exists($field, $values)) {
1131           if (!array_key_exists('address', $params)) {
1132             $params['address'] = [];
1133           }
1134           $params['address'][$addressCnt][$field] = $values[$field];
1135         }
1136       }
1137
1138       if ($addressCnt == 1) {
1139
1140         $params['address'][$addressCnt]['is_primary'] = TRUE;
1141       }
1142       return TRUE;
1143     }
1144
1145     if (isset($values['note'])) {
1146       // add a note field
1147       if (!isset($params['note'])) {
1148         $params['note'] = [];
1149       }
1150       $noteBlock = count($params['note']) + 1;
1151
1152       $params['note'][$noteBlock] = [];
1153       if (!isset($fields['Note'])) {
1154         $fields['Note'] = CRM_Core_DAO_Note::fields();
1155       }
1156
1157       // get the current logged in civicrm user
1158       $session = CRM_Core_Session::singleton();
1159       $userID = $session->get('userID');
1160
1161       if ($userID) {
1162         $values['contact_id'] = $userID;
1163       }
1164
1165       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1166
1167       return TRUE;
1168     }
1169
1170     // Check for custom field values
1171
1172     if (empty($fields['custom'])) {
1173       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1174         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1175       );
1176     }
1177
1178     foreach ($values as $key => $value) {
1179       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1180         // check if it's a valid custom field id
1181
1182         if (!array_key_exists($customFieldID, $fields['custom'])) {
1183           return civicrm_api3_create_error('Invalid custom field ID');
1184         }
1185         else {
1186           $params[$key] = $value;
1187         }
1188       }
1189     }
1190   }
1191
1192   /**
1193    * Parse a field which could be represented by a label or name value rather than the DB value.
1194    *
1195    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1196    *
1197    * but if not available then see if we have a label that can be converted to a name.
1198    *
1199    * @param string|int|null $submittedValue
1200    * @param array $fieldSpec
1201    *   Metadata for the field
1202    *
1203    * @return mixed
1204    */
1205   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1206     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1207     if (!isset($fieldSpec['bao'])) {
1208       return $submittedValue;
1209     }
1210     /* @var \CRM_Core_DAO $bao */
1211     $bao = $fieldSpec['bao'];
1212     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1213     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1214     if (isset($nameOptions[$submittedValue])) {
1215       return $submittedValue;
1216     }
1217     if (in_array($submittedValue, $nameOptions)) {
1218       return array_search($submittedValue, $nameOptions, TRUE);
1219     }
1220
1221     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1222     if (isset($labelOptions[$submittedValue])) {
1223       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1224     }
1225     return '';
1226   }
1227
1228   /**
1229    * This is code extracted from 4 places where this exact snippet was being duplicated.
1230    *
1231    * FIXME: Extracting this was a first step, but there's also
1232    *  1. Inconsistency in the way other select options are handled.
1233    *     Contribution adds handling for Select/Radio/Autocomplete
1234    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1235    *     Membership is missing all of it
1236    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1237    *
1238    * @param $customFieldID
1239    * @param $value
1240    * @param $fieldType
1241    * @return array
1242    */
1243   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1244     $mulValues = explode(',', $value);
1245     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1246     $values = [];
1247     foreach ($mulValues as $v1) {
1248       foreach ($customOption as $customValueID => $customLabel) {
1249         $customValue = $customLabel['value'];
1250         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1251           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1252         ) {
1253           $values[] = $customValue;
1254         }
1255       }
1256     }
1257     return $values;
1258   }
1259
1260   /**
1261    * Validate that the field requirements are met in the params.
1262    *
1263    * @param array $requiredFields
1264    * @param array $params
1265    *   An array of required fields (fieldName => label)
1266    *   - note this follows the and / or array nesting we see in permission checks
1267    *   eg.
1268    *   [
1269    *     'email' => ts('Email'),
1270    *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1271    *   ]
1272    *   Means 'email' OR 'first_name AND 'last_name'.
1273    * @param string $prefixString
1274    *
1275    * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1276    */
1277   protected function validateRequiredFields(array $requiredFields, array $params, $prefixString = ''): void {
1278     $missingFields = [];
1279     foreach ($requiredFields as $key => $required) {
1280       if (!is_array($required)) {
1281         $importParameter = $params[$key] ?? [];
1282         if (!is_array($importParameter)) {
1283           if (!empty($importParameter)) {
1284             return;
1285           }
1286         }
1287         else {
1288           foreach ($importParameter as $locationValues) {
1289             if (!empty($locationValues[$key])) {
1290               return;
1291             }
1292           }
1293         }
1294
1295         $missingFields[$key] = $required;
1296       }
1297       else {
1298         foreach ($required as $field => $label) {
1299           if (empty($params[$field])) {
1300             $missing[$field] = $label;
1301           }
1302         }
1303         if (empty($missing)) {
1304           return;
1305         }
1306         $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1307       }
1308     }
1309     throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1310   }
1311
1312   /**
1313    * Get the field value, transformed by metadata.
1314    *
1315    * @param string $fieldName
1316    * @param string|int $importedValue
1317    *   Value as it came in from the datasource.
1318    *
1319    * @return string|array|bool|int
1320    * @throws \API_Exception
1321    */
1322   protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1323     if (empty($importedValue)) {
1324       return $importedValue;
1325     }
1326     $fieldMetadata = $this->getFieldMetadata($fieldName);
1327     if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) {
1328       $values = [];
1329       foreach (explode(',', $importedValue) as $value) {
1330         $values[] = $this->getTransformedFieldValue($fieldName, trim($value));
1331       }
1332       return $values;
1333     }
1334     if ($fieldName === 'url') {
1335       return CRM_Utils_Rule::url($importedValue) ? $importedValue : 'invalid_import_value';
1336     }
1337
1338     if ($fieldName === 'email') {
1339       return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value';
1340     }
1341
1342     if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) {
1343       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1344     }
1345     if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) {
1346       return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value';
1347     }
1348     if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) {
1349       $value = CRM_Utils_String::strtoboolstr($importedValue);
1350       if ($value !== FALSE) {
1351         return (bool) $value;
1352       }
1353       return 'invalid_import_value';
1354     }
1355     if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) {
1356       $value = CRM_Utils_Date::formatDate($importedValue, (int) $this->getSubmittedValue('dateFormats'));
1357       return $value ?: 'invalid_import_value';
1358     }
1359     $options = $this->getFieldOptions($fieldName);
1360     if ($options !== FALSE) {
1361       if ($this->isAmbiguous($fieldName, $importedValue)) {
1362         // We can't transform it at this stage. Perhaps later we can with
1363         // other information such as country.
1364         return $importedValue;
1365       }
1366
1367       $comparisonValue = $this->getComparisonValue($importedValue);
1368       return $options[$comparisonValue] ?? 'invalid_import_value';
1369     }
1370     if (!empty($fieldMetadata['FKClassName']) || !empty($fieldMetadata['pseudoconstant']['prefetch'])) {
1371       // @todo - make this generic - for fields where getOptions doesn't fetch
1372       // getOptions does not retrieve these fields with high potential results
1373       if ($fieldName === 'event_id') {
1374         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1375           $event = Event::get()->addClause('OR', ['title', '=', $importedValue], ['id', '=', $importedValue])->addSelect('id')->execute()->first();
1376           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $event['id'] ?? FALSE;
1377         }
1378         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1379       }
1380       if ($fieldMetadata['name'] === 'campaign_id') {
1381         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1382           $campaign = Campaign::get()->addClause('OR', ['title', '=', $importedValue], ['name', '=', $importedValue])->addSelect('id')->execute()->first();
1383           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $campaign['id'] ?? FALSE;
1384         }
1385         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1386       }
1387     }
1388     if ($fieldMetadata['type'] === CRM_Utils_Type::T_INT) {
1389       // We have resolved the options now so any remaining ones should be integers.
1390       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1391     }
1392     return $importedValue;
1393   }
1394
1395   /**
1396    * @param string $fieldName
1397    *
1398    * @return false|array
1399    *
1400    * @throws \API_Exception
1401    */
1402   protected function getFieldOptions(string $fieldName) {
1403     return $this->getFieldMetadata($fieldName, TRUE)['options'];
1404   }
1405
1406   /**
1407    * Get the metadata for the field.
1408    *
1409    * @param string $fieldName
1410    * @param bool $loadOptions
1411    * @param bool $limitToContactType
1412    *   Only show fields for the type to import (not appropriate when looking up
1413    *   related contact fields).
1414    *
1415    * @return array
1416    *
1417    * @noinspection PhpDocMissingThrowsInspection
1418    * @noinspection PhpUnhandledExceptionInspection
1419    */
1420   protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1421
1422     $fieldMap = $this->getOddlyMappedMetadataFields();
1423     $fieldMapName = empty($fieldMap[$fieldName]) ? $fieldName : $fieldMap[$fieldName];
1424
1425     // This whole business of only loading metadata for one type when we actually need it for all is ... dubious.
1426     if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) {
1427       if ($loadOptions || !$limitToContactType) {
1428         $this->importableFieldsMetadata[$fieldMapName] = CRM_Contact_BAO_Contact::importableFields('All')[$fieldMapName];
1429       }
1430     }
1431
1432     $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName];
1433     if ($loadOptions && !isset($fieldMetadata['options'])) {
1434       if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') {
1435         // Probably already loaded and also supports abbreviations - eg. NSW.
1436         // Supporting for core AND custom state fields is more consistent.
1437         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id');
1438         return $this->importableFieldsMetadata[$fieldMapName];
1439       }
1440       if (($fieldMetadata['data_type'] ?? '') === 'Country') {
1441         // Probably already loaded and also supports abbreviations - eg. NSW.
1442         // Supporting for core AND custom state fields is more consistent.
1443         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('country_id');
1444         return $this->importableFieldsMetadata[$fieldMapName];
1445       }
1446       $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;
1447
1448       if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
1449         $options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
1450           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1451           'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
1452           'select' => ['options'],
1453         ])->first()['options'];
1454       }
1455       else {
1456         if (!empty($fieldMetadata['custom_group_id'])) {
1457           $customField = CustomField::get(FALSE)
1458             ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
1459             ->addSelect('name', 'custom_group_id.name')
1460             ->execute()
1461             ->first();
1462           $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
1463         }
1464         $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1465           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1466           'where' => [['name', '=', $optionFieldName]],
1467           'select' => ['options'],
1468         ])->first()['options'];
1469       }
1470       if (is_array($options)) {
1471         // We create an array of the possible variants - notably including
1472         // name AND label as either might be used. We also lower case before checking
1473         $values = [];
1474         foreach ($options as $option) {
1475           $idKey = $this->getComparisonValue($option['id']);
1476           $values[$idKey] = $option['id'];
1477           foreach (['name', 'label', 'abbr'] as $key) {
1478             $optionValue = $this->getComparisonValue($option[$key] ?? '');
1479             if ($optionValue !== '') {
1480               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
1481                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
1482                   $this->ambiguousOptions[$fieldName][$optionValue] = [$values[$optionValue]];
1483                 }
1484                 $this->ambiguousOptions[$fieldName][$optionValue][] = $option['id'];
1485               }
1486               else {
1487                 $values[$optionValue] = $option['id'];
1488               }
1489             }
1490           }
1491         }
1492         $this->importableFieldsMetadata[$fieldMapName]['options'] = $values;
1493       }
1494       else {
1495         $this->importableFieldsMetadata[$fieldMapName]['options'] = $options ?: FALSE;
1496       }
1497       return $this->importableFieldsMetadata[$fieldMapName];
1498     }
1499     return $fieldMetadata;
1500   }
1501
1502   /**
1503    * Get the field metadata for fields to be be offered to match the contact.
1504    *
1505    * @return array
1506    * @noinspection PhpDocMissingThrowsInspection
1507    */
1508   protected function getContactMatchingFields(): array {
1509     $contactFields = CRM_Contact_BAO_Contact::importableFields($this->getContactType(), NULL);
1510     $fields = ['external_identifier' => $contactFields['external_identifier']];
1511     $fields['external_identifier']['title'] .= ' (match to contact)';
1512     // Using new Dedupe rule.
1513     $ruleParams = [
1514       'contact_type' => $this->getContactType(),
1515       'used' => $this->getSubmittedValue('dedupe_rule_id') ?? 'Unsupervised',
1516     ];
1517     $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
1518
1519     if (is_array($fieldsArray)) {
1520       foreach ($fieldsArray as $value) {
1521         $customFieldId = CRM_Core_DAO::getFieldValue('CRM_Core_DAO_CustomField',
1522           $value,
1523           'id',
1524           'column_name'
1525         );
1526         $value = trim($customFieldId ? 'custom_' . $customFieldId : $value);
1527         $fields[$value] = $contactFields[$value] ?? NULL;
1528         $title = $fields[$value]['title'] . ' (match to contact)';
1529         $fields[$value]['title'] = $title;
1530       }
1531     }
1532     return $fields;
1533   }
1534
1535   /**
1536    * @param $customFieldID
1537    * @param $value
1538    * @param array $fieldMetaData
1539    * @param $dateType
1540    *
1541    * @return ?string
1542    */
1543   protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?string {
1544     /* validate the data against the CF type */
1545
1546     if ($value) {
1547       $dataType = $fieldMetaData['data_type'];
1548       $htmlType = $fieldMetaData['html_type'];
1549       $isSerialized = CRM_Core_BAO_CustomField::isSerialized($fieldMetaData);
1550       if ($dataType === 'Date') {
1551         $params = ['date_field' => $value];
1552         if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, 'date_field')) {
1553           return NULL;
1554         }
1555         return $fieldMetaData['label'];
1556       }
1557       elseif ($dataType === 'Boolean') {
1558         if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1559           return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1560         }
1561       }
1562       // need not check for label filed import
1563       $selectHtmlTypes = [
1564         'CheckBox',
1565         'Select',
1566         'Radio',
1567       ];
1568       if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1569         $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1570         if (!$valid) {
1571           return $fieldMetaData['label'];
1572         }
1573       }
1574
1575       // check for values for custom fields for checkboxes and multiselect
1576       if ($isSerialized && $dataType != 'ContactReference') {
1577         $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1578         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1579         foreach ($mulValues as $v1) {
1580
1581           $flag = FALSE;
1582           foreach ($customOption as $v2) {
1583             if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1584               $flag = TRUE;
1585             }
1586           }
1587
1588           if (!$flag) {
1589             return $fieldMetaData['label'];
1590           }
1591         }
1592       }
1593       elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1594         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1595         $flag = FALSE;
1596         foreach ($customOption as $v2) {
1597           if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1598             $flag = TRUE;
1599           }
1600         }
1601         if (!$flag) {
1602           return $fieldMetaData['label'];
1603         }
1604       }
1605     }
1606
1607     return NULL;
1608   }
1609
1610   /**
1611    * Get the entity for the given field.
1612    *
1613    * @param string $fieldName
1614    *
1615    * @return mixed|null
1616    * @throws \API_Exception
1617    */
1618   protected function getFieldEntity(string $fieldName) {
1619     if ($fieldName === 'do_not_import') {
1620       return NULL;
1621     }
1622     if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) {
1623       return 'Contact';
1624     }
1625     $metadata = $this->getFieldMetadata($fieldName);
1626     if (!isset($metadata['entity'])) {
1627       return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ? 'Contact' : $metadata['extends'];
1628     }
1629
1630     // Our metadata for these is fugly. Handling the fugliness during retrieval.
1631     if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1632       return 'Address';
1633     }
1634     return $metadata['entity'];
1635   }
1636
1637   /**
1638    * Validate the import file, updating the import table with results.
1639    *
1640    * @throws \API_Exception
1641    * @throws \CRM_Core_Exception
1642    */
1643   public function validate(): void {
1644     $dataSource = $this->getDataSourceObject();
1645     while ($row = $dataSource->getRow()) {
1646       try {
1647         $rowNumber = $row['_id'];
1648         $values = array_values($row);
1649         $this->validateValues($values);
1650         $this->setImportStatus($rowNumber, 'NEW', '');
1651       }
1652       catch (CRM_Core_Exception $e) {
1653         $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
1654       }
1655     }
1656   }
1657
1658   /**
1659    * Validate the import values.
1660    *
1661    * The values array represents a row in the datasource.
1662    *
1663    * @param array $values
1664    *
1665    * @throws \API_Exception
1666    * @throws \CRM_Core_Exception
1667    */
1668   public function validateValues(array $values): void {
1669     $params = $this->getMappedRow($values);
1670     $this->validateParams($params);
1671   }
1672
1673   /**
1674    * @param array $params
1675    *
1676    * @throws \CRM_Core_Exception
1677    */
1678   protected function validateParams(array $params): void {
1679     if (empty($params['id'])) {
1680       $this->validateRequiredFields($this->getRequiredFields(), $params);
1681     }
1682     $errors = [];
1683     foreach ($params as $key => $value) {
1684       $errors = array_merge($this->getInvalidValues($value, $key), $errors);
1685     }
1686     if ($errors) {
1687       throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors));
1688     }
1689   }
1690
1691   /**
1692    * Search the value for the string 'invalid_import_value'.
1693    *
1694    * If the string is found it indicates the fields was rejected
1695    * during `getTransformedValue` as not having valid data.
1696    *
1697    * @param string|array|int $value
1698    * @param string $key
1699    * @param string $prefixString
1700    *
1701    * @return array
1702    */
1703   protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
1704     $errors = [];
1705     if ($value === 'invalid_import_value') {
1706       if (!is_numeric($key)) {
1707         $metadata = $this->getFieldMetadata($key);
1708         $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1709       }
1710       else {
1711         // Numeric key suggests we are drilling into option values
1712         $errors[] = TRUE;
1713       }
1714     }
1715     elseif (is_array($value)) {
1716       foreach ($value as $innerKey => $innerValue) {
1717         $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1718         if ($result === [TRUE]) {
1719           $metadata = $this->getFieldMetadata($key);
1720           $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1721         }
1722         elseif (!empty($result)) {
1723           $errors = array_merge($result, $errors);
1724         }
1725       }
1726     }
1727     return array_filter($errors);
1728   }
1729
1730   /**
1731    * Get the available countries.
1732    *
1733    * If the site is not configured with a restriction then all countries are valid
1734    * but otherwise only a select array are.
1735    *
1736    * @return array|false
1737    *   FALSE indicates no restrictions.
1738    */
1739   protected function getAvailableCountries() {
1740     if ($this->availableCountries === NULL) {
1741       $availableCountries = Civi::settings()->get('countryLimit');
1742       $this->availableCountries = !empty($availableCountries) ? array_fill_keys($availableCountries, TRUE) : FALSE;
1743     }
1744     return $this->availableCountries;
1745   }
1746
1747   /**
1748    * Get the metadata field for which importable fields does not key the actual field name.
1749    *
1750    * @return string[]
1751    */
1752   protected function getOddlyMappedMetadataFields(): array {
1753     return [
1754       'country_id' => 'country',
1755       'state_province_id' => 'state_province',
1756       'county_id' => 'county',
1757       'email_greeting_id' => 'email_greeting',
1758       'postal_greeting_id' => 'postal_greeting',
1759       'addressee_id' => 'addressee',
1760     ];
1761   }
1762
1763   /**
1764    * Get the default country for the site.
1765    *
1766    * @return int
1767    */
1768   protected function getSiteDefaultCountry(): int {
1769     if (!isset($this->siteDefaultCountry)) {
1770       $this->siteDefaultCountry = (int) Civi::settings()->get('defaultContactCountry');
1771     }
1772     return $this->siteDefaultCountry;
1773   }
1774
1775   /**
1776    * Is the option ambiguous.
1777    *
1778    * @param string $fieldName
1779    * @param string $importedValue
1780    */
1781   protected function isAmbiguous(string $fieldName, $importedValue): bool {
1782     return !empty($this->ambiguousOptions[$fieldName][$this->getComparisonValue($importedValue)]);
1783   }
1784
1785   /**
1786    * Get the civicrm_mapping_field appropriate layout for the mapper input.
1787    *
1788    * For simple parsers (not contribution or contact) the input looks like
1789    * ['first_name', 'custom_32']
1790    * and it is converted to
1791    *
1792    *  ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5],
1793    *
1794    * @param array $fieldMapping
1795    * @param int $mappingID
1796    * @param int $columnNumber
1797    *
1798    * @return array
1799    */
1800   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
1801     return [
1802       'name' => $fieldMapping[0],
1803       'mapping_id' => $mappingID,
1804       'column_number' => $columnNumber,
1805     ];
1806   }
1807
1808   /**
1809    * The initializer code, called before the processing
1810    *
1811    * @return void
1812    */
1813   public function init() {
1814     // Force re-load of user job.
1815     unset($this->userJob);
1816     $this->setFieldMetadata();
1817   }
1818
1819   /**
1820    * @param array $mappedField
1821    *   Field detail as would be saved in field_mapping table
1822    *   or as returned from getMappingFieldFromMapperInput
1823    *
1824    * @return string
1825    * @throws \API_Exception
1826    */
1827   public function getMappedFieldLabel(array $mappedField): string {
1828     // doNotImport is on it's way out - skip fields will be '' once all is done.
1829     if ($mappedField['name'] === 'doNotImport') {
1830       return '';
1831     }
1832     $this->setFieldMetadata();
1833     $metadata = $this->getFieldMetadata($mappedField['name']);
1834     return $metadata['html']['label'] ?? $metadata['title'];
1835   }
1836
1837   /**
1838    * Get the row from the csv mapped to our parameters.
1839    *
1840    * @param array $values
1841    *
1842    * @return array
1843    * @throws \API_Exception
1844    */
1845   public function getMappedRow(array $values): array {
1846     $params = [];
1847     foreach ($this->getFieldMappings() as $i => $mappedField) {
1848       if ($mappedField['name'] === 'do_not_import') {
1849         continue;
1850       }
1851       if ($mappedField['name']) {
1852         $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
1853       }
1854     }
1855     return $params;
1856   }
1857
1858   /**
1859    * Get the field mappings for the import.
1860    *
1861    * This is the same format as saved in civicrm_mapping_field except
1862    * that location_type_id = 'Primary' rather than empty where relevant.
1863    * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
1864    *
1865    * @return array
1866    * @throws \API_Exception
1867    */
1868   protected function getFieldMappings(): array {
1869     $mappedFields = [];
1870     $mapper = $this->getSubmittedValue('mapper');
1871     foreach ($mapper as $i => $mapperRow) {
1872       $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
1873       // Just for clarity since 0 is a pseudo-value
1874       unset($mappedField['mapping_id']);
1875       $mappedFields[] = $mappedField;
1876     }
1877     return $mappedFields;
1878   }
1879
1880   /**
1881    * Run import.
1882    *
1883    * @param \CRM_Queue_TaskContext $taskContext
1884    *
1885    * @param int $userJobID
1886    * @param int $limit
1887    *
1888    * @return bool
1889    * @throws \API_Exception
1890    * @throws \CRM_Core_Exception
1891    */
1892   public static function runImport($taskContext, $userJobID, $limit) {
1893     $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('type_id')->execute()->first();
1894     $parserClass = NULL;
1895     foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
1896       if ($userJob['type_id'] === $userJobType['id']) {
1897         $parserClass = $userJobType['class'];
1898       }
1899     }
1900     /* @var \CRM_Import_Parser $parser */
1901     $parser = new $parserClass();
1902     $parser->setUserJobID($userJobID);
1903     // Not sure if we still need to init....
1904     $parser->init();
1905     $dataSource = $parser->getDataSourceObject();
1906     $dataSource->setStatuses(['new']);
1907     $dataSource->setLimit($limit);
1908
1909     while ($row = $dataSource->getRow()) {
1910       $values = array_values($row);
1911       $parser->import($values);
1912     }
1913     $parser->doPostImportActions();
1914     return TRUE;
1915   }
1916
1917   /**
1918    * Check if an error in custom data.
1919    *
1920    * @deprecated all of this is duplicated if getTransformedValue is used.
1921    *
1922    * @param array $params
1923    * @param string $errorMessage
1924    *   A string containing all the error-fields.
1925    *
1926    * @param null $csType
1927    */
1928   public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) {
1929     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1930     $errors = [];
1931
1932     if (!empty($params['contact_sub_type'])) {
1933       $csType = $params['contact_sub_type'] ?? NULL;
1934     }
1935
1936     if (empty($params['contact_type'])) {
1937       $params['contact_type'] = 'Individual';
1938     }
1939
1940     // get array of subtypes - CRM-18708
1941     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1942       $csType = $this->getSubtypes($params['contact_type']);
1943     }
1944
1945     if (is_array($csType)) {
1946       // fetch custom fields for every subtype and add it to $customFields array
1947       // CRM-18708
1948       $customFields = [];
1949       foreach ($csType as $cType) {
1950         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1951       }
1952     }
1953     else {
1954       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1955     }
1956
1957     foreach ($params as $key => $value) {
1958       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1959         //For address custom fields, we do get actual custom field value as an inner array of
1960         //values so need to modify
1961         if (!array_key_exists($customFieldID, $customFields)) {
1962           return ts('field ID');
1963         }
1964         /* check if it's a valid custom field id */
1965         $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType);
1966       }
1967     }
1968     if ($errors) {
1969       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', array_filter($errors));
1970     }
1971   }
1972
1973   /**
1974    * get subtypes given the contact type
1975    *
1976    * @param string $contactType
1977    * @return array $subTypes
1978    */
1979   protected function getSubtypes($contactType) {
1980     $subTypes = [];
1981     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
1982
1983     if (count($types) > 0) {
1984       foreach ($types as $type) {
1985         $subTypes[] = $type['name'];
1986       }
1987     }
1988     return $subTypes;
1989   }
1990
1991   /**
1992    * Update the status of the import row to reflect the processing outcome.
1993    *
1994    * @param int $id
1995    * @param string $status
1996    * @param string $message
1997    * @param int|null $entityID
1998    *   Optional created entity ID
1999    * @param array $additionalFields
2000    *  Additional fields to be tracked
2001    * @param array $createdContactIDs
2002    *
2003    * @noinspection PhpDocMissingThrowsInspection
2004    * @noinspection PhpUnhandledExceptionInspection
2005    */
2006   protected function setImportStatus(int $id, string $status, string $message = '', ?int $entityID = NULL, $additionalFields = [], $createdContactIDs = []): void {
2007     foreach ($createdContactIDs as $createdContactID) {
2008       // Store any created contacts for post_actions like tag or add to group.
2009       // These are done on a 'per-batch' status in processPorstActions
2010       // so holding in a property is OK.
2011       $this->createdContacts[$createdContactID] = $createdContactID;
2012     }
2013     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields);
2014   }
2015
2016   /**
2017    * Convert any given date string to default date array.
2018    *
2019    * @param array $params
2020    *   Has given date-format.
2021    * @param array $formatted
2022    *   Store formatted date in this array.
2023    * @param int $dateType
2024    *   Type of date.
2025    * @param string $dateParam
2026    *   Index of params.
2027    */
2028   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2029     //fix for CRM-2687
2030     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
2031     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
2032   }
2033
2034   /**
2035    * Get the value to use for option comparison purposes.
2036    *
2037    * We do a case-insensitive comparison, also swapping ’ for '
2038    * which has at least one known usage (Côte d’Ivoire).
2039    *
2040    * Note we do this to both sides of the comparison.
2041    *
2042    * @param int|string|false|null $importedValue
2043    *
2044    * @return false|int|string|null
2045    */
2046   protected function getComparisonValue($importedValue) {
2047     return is_numeric($importedValue) ? $importedValue : mb_strtolower(str_replace('’', "'", $importedValue));
2048   }
2049
2050 }