CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Campaign;
  13 use Civi\Api4\CustomField;
  14 use Civi\Api4\Event;
  15 use Civi\Api4\UserJob;
  16
  17 /**
  18  *
  19  * @package CRM
  20  * @copyright CiviCRM LLC https://civicrm.org/licensing
  21  */
  22 abstract class CRM_Import_Parser {
  23   /**
  24    * Settings
  25    */
  26   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  27
  28   /**
  29    * Return codes
  30    */
  31   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  32
  33   /**
  34    * Parser modes
  35    */
  36   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  37
  38   /**
  39    * Codes for duplicate record handling
  40    */
  41   const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  42
  43   /**
  44    * Contact types
  45    */
  46   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  47
  48   /**
  49    * User job id.
  50    *
  51    * This is the primary key of the civicrm_user_job table which is used to
  52    * track the import.
  53    *
  54    * @var int
  55    */
  56   protected $userJobID;
  57
  58   /**
  59    * The user job in use.
  60    *
  61    * @var array
  62    */
  63   protected $userJob;
  64
  65   /**
  66    * Potentially ambiguous options.
  67    *
  68    * For example 'UT' is a state in more than one country.
  69    *
  70    * @var array
  71    */
  72   protected $ambiguousOptions = [];
  73
  74   /**
  75    * States to country mapping.
  76    *
  77    * @var array
  78    */
  79   protected $statesByCountry = [];
  80
  81   /**
  82    * @return int|null
  83    */
  84   public function getUserJobID(): ?int {
  85     return $this->userJobID;
  86   }
  87
  88   /**
  89    * Ids of contacts created this iteration.
  90    *
  91    * @var array
  92    */
  93   protected $createdContacts = [];
  94
  95   /**
  96    * Set user job ID.
  97    *
  98    * @param int $userJobID
  99    *
 100    * @return self
 101    */
 102   public function setUserJobID(int $userJobID): self {
 103     $this->userJobID = $userJobID;
 104     return $this;
 105   }
 106
 107   /**
 108    * Countries that the site is restricted to
 109    *
 110    * @var array|false
 111    */
 112   private $availableCountries;
 113
 114   /**
 115    *
 116    * @return array
 117    */
 118   public function getTrackingFields(): array {
 119     return [];
 120   }
 121
 122   /**
 123    * Get User Job.
 124    *
 125    * API call to retrieve the userJob row.
 126    *
 127    * @return array
 128    *
 129    * @throws \API_Exception
 130    */
 131   protected function getUserJob(): array {
 132     if (empty($this->userJob)) {
 133       $this->userJob = UserJob::get()
 134         ->addWhere('id', '=', $this->getUserJobID())
 135         ->execute()
 136         ->first();
 137     }
 138     return $this->userJob;
 139   }
 140
 141   /**
 142    * Get the relevant datasource object.
 143    *
 144    * @return \CRM_Import_DataSource|null
 145    *
 146    * @throws \API_Exception
 147    */
 148   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 149     $className = $this->getSubmittedValue('dataSource');
 150     if ($className) {
 151       /* @var CRM_Import_DataSource $dataSource */
 152       return new $className($this->getUserJobID());
 153     }
 154     return NULL;
 155   }
 156
 157   /**
 158    * Get the submitted value, as stored on the user job.
 159    *
 160    * @param string $fieldName
 161    *
 162    * @return mixed
 163    *
 164    * @noinspection PhpDocMissingThrowsInspection
 165    * @noinspection PhpUnhandledExceptionInspection
 166    */
 167   protected function getSubmittedValue(string $fieldName) {
 168     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 169   }
 170
 171   /**
 172    * Has the import completed.
 173    *
 174    * @return bool
 175    *
 176    * @throws \API_Exception
 177    * @throws \CRM_Core_Exception
 178    */
 179   public function isComplete() :bool {
 180     return $this->getDataSourceObject()->isCompleted();
 181   }
 182
 183   /**
 184    * Get configured contact type.
 185    *
 186    * @return string
 187    */
 188   protected function getContactType(): string {
 189     if (!$this->_contactType) {
 190       $contactTypeMapping = [
 191         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 192         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 193         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 194       ];
 195       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 196     }
 197     return $this->_contactType;
 198   }
 199
 200   /**
 201    * Get configured contact type.
 202    *
 203    * @return string|null
 204    */
 205   public function getContactSubType(): ?string {
 206     if (!$this->_contactSubType) {
 207       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 208     }
 209     return $this->_contactSubType;
 210   }
 211
 212   /**
 213    * Total number of non empty lines
 214    * @var int
 215    */
 216   protected $_totalCount;
 217
 218   /**
 219    * Running total number of valid lines
 220    * @var int
 221    */
 222   protected $_validCount;
 223
 224   /**
 225    * Running total number of invalid rows
 226    * @var int
 227    */
 228   protected $_invalidRowCount;
 229
 230   /**
 231    * Maximum number of non-empty/comment lines to process
 232    *
 233    * @var int
 234    */
 235   protected $_maxLinesToProcess;
 236
 237   /**
 238    * Array of error lines, bounded by MAX_ERROR
 239    * @var array
 240    */
 241   protected $_errors;
 242
 243   /**
 244    * Total number of duplicate (from database) lines
 245    * @var int
 246    */
 247   protected $_duplicateCount;
 248
 249   /**
 250    * Array of duplicate lines
 251    * @var array
 252    */
 253   protected $_duplicates;
 254
 255   /**
 256    * Maximum number of warnings to store
 257    * @var int
 258    */
 259   protected $_maxWarningCount = self::MAX_WARNINGS;
 260
 261   /**
 262    * Array of warning lines, bounded by MAX_WARNING
 263    * @var array
 264    */
 265   protected $_warnings;
 266
 267   /**
 268    * Array of all the fields that could potentially be part
 269    * of this import process
 270    * @var array
 271    */
 272   protected $_fields;
 273
 274   /**
 275    * Metadata for all available fields, keyed by unique name.
 276    *
 277    * This is intended to supercede $_fields which uses a special sauce format which
 278    * importableFieldsMetadata uses the standard getfields type format.
 279    *
 280    * @var array
 281    */
 282   protected $importableFieldsMetadata = [];
 283
 284   /**
 285    * Get metadata for all importable fields in std getfields style format.
 286    *
 287    * @return array
 288    */
 289   public function getImportableFieldsMetadata(): array {
 290     return $this->importableFieldsMetadata;
 291   }
 292
 293   /**
 294    * Set metadata for all importable fields in std getfields style format.
 295    *
 296    * @param array $importableFieldsMetadata
 297    */
 298   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 299     $this->importableFieldsMetadata = $importableFieldsMetadata;
 300   }
 301
 302   /**
 303    * Gets the fields available for importing in a key-name, title format.
 304    *
 305    * @return array
 306    *   eg. ['first_name' => 'First Name'.....]
 307    *
 308    * @throws \API_Exception
 309    *
 310    * @todo - we are constructing the metadata before we
 311    * have set the contact type so we re-do it here.
 312    *
 313    * Once we have cleaned up the way the mapper is handled
 314    * we can ditch all the existing _construct parameters in favour
 315    * of just the userJobID - there are current open PRs towards this end.
 316    */
 317   public function getAvailableFields(): array {
 318     $this->setFieldMetadata();
 319     $return = [];
 320     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 321       if ($name === 'id' && $this->isSkipDuplicates()) {
 322         // Duplicates are being skipped so id matching is not availble.
 323         continue;
 324       }
 325       $return[$name] = $field['html']['label'] ?? $field['title'];
 326     }
 327     return $return;
 328   }
 329
 330   /**
 331    * Did the user specify duplicates should be skipped and not imported.
 332    *
 333    * @return bool
 334    *
 335    * @throws \API_Exception
 336    */
 337   protected function isSkipDuplicates(): bool {
 338     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 339   }
 340
 341   /**
 342    * Is this a case where the user has opted to update existing contacts.
 343    *
 344    * @return bool
 345    */
 346   protected function isUpdateExisting(): bool {
 347     return in_array((int) $this->getSubmittedValue('onDuplicate'), [
 348       CRM_Import_Parser::DUPLICATE_UPDATE,
 349       CRM_Import_Parser::DUPLICATE_FILL,
 350     ], TRUE);
 351   }
 352
 353   /**
 354    * Did the user specify duplicates checking should be skipped, resulting in possible duplicate contacts.
 355    *
 356    * Note we still need to check for external_identifier as it will hard-fail
 357    * if we duplicate.
 358    *
 359    * @return bool
 360    */
 361   protected function isIgnoreDuplicates(): bool {
 362     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_NOCHECK;
 363   }
 364
 365   /**
 366    * Did the user specify duplicates should be filled with missing data.
 367    *
 368    * @return bool
 369    */
 370   protected function isFillDuplicates(): bool {
 371     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_FILL;
 372   }
 373
 374   /**
 375    * Array of the fields that are actually part of the import process
 376    * the position in the array also dictates their position in the import
 377    * file
 378    * @var array
 379    */
 380   protected $_activeFields = [];
 381
 382   /**
 383    * Cache the count of active fields
 384    *
 385    * @var int
 386    */
 387   protected $_activeFieldCount;
 388
 389   /**
 390    * Cache of preview rows
 391    *
 392    * @var array
 393    */
 394   protected $_rows;
 395
 396   /**
 397    * Filename of error data
 398    *
 399    * @var string
 400    */
 401   protected $_errorFileName;
 402
 403   /**
 404    * Filename of duplicate data
 405    *
 406    * @var string
 407    */
 408   protected $_duplicateFileName;
 409
 410   /**
 411    * Contact type
 412    *
 413    * @var string
 414    */
 415   public $_contactType;
 416
 417   /**
 418    * @param string $contactType
 419    *
 420    * @return CRM_Import_Parser
 421    */
 422   public function setContactType(string $contactType): CRM_Import_Parser {
 423     $this->_contactType = $contactType;
 424     return $this;
 425   }
 426
 427   /**
 428    * Contact sub-type
 429    *
 430    * @var int|null
 431    */
 432   public $_contactSubType;
 433
 434   /**
 435    * @param int|null $contactSubType
 436    *
 437    * @return self
 438    */
 439   public function setContactSubType(?int $contactSubType): self {
 440     $this->_contactSubType = $contactSubType;
 441     return $this;
 442   }
 443
 444   /**
 445    * Class constructor.
 446    */
 447   public function __construct() {
 448     $this->_maxLinesToProcess = 0;
 449   }
 450
 451   /**
 452    * Set and validate field values.
 453    *
 454    * @param array $elements
 455    *   array.
 456    */
 457   public function setActiveFieldValues($elements): void {
 458     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 459     for ($i = 0; $i < $maxCount; $i++) {
 460       $this->_activeFields[$i]->setValue($elements[$i]);
 461     }
 462
 463     // reset all the values that we did not have an equivalent import element
 464     for (; $i < $this->_activeFieldCount; $i++) {
 465       $this->_activeFields[$i]->resetValue();
 466     }
 467   }
 468
 469   /**
 470    * Format the field values for input to the api.
 471    *
 472    * @return array
 473    *   (reference) associative array of name/value pairs
 474    */
 475   public function &getActiveFieldParams() {
 476     $params = [];
 477     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 478       if (isset($this->_activeFields[$i]->_value)
 479         && !isset($params[$this->_activeFields[$i]->_name])
 480         && !isset($this->_activeFields[$i]->_related)
 481       ) {
 482
 483         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 484       }
 485     }
 486     return $params;
 487   }
 488
 489   /**
 490    * Add progress bar to the import process. Calculates time remaining, status etc.
 491    *
 492    * @param $statusID
 493    *   status id of the import process saved in $config->uploadDir.
 494    * @param bool $startImport
 495    *   True when progress bar is to be initiated.
 496    * @param $startTimestamp
 497    *   Initial timestamp when the import was started.
 498    * @param $prevTimestamp
 499    *   Previous timestamp when this function was last called.
 500    * @param $totalRowCount
 501    *   Total number of rows in the import file.
 502    *
 503    * @return NULL|$currTimestamp
 504    */
 505   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 506     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 507
 508     if ($startImport) {
 509       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 510       //do not force the browser to display the save dialog, CRM-7640
 511       $contents = json_encode([0, $status]);
 512       file_put_contents($statusFile, $contents);
 513     }
 514     else {
 515       $rowCount = $this->_rowCount ?? $this->_lineCount;
 516       $currTimestamp = time();
 517       $time = ($currTimestamp - $prevTimestamp);
 518       $recordsLeft = $totalRowCount - $rowCount;
 519       if ($recordsLeft < 0) {
 520         $recordsLeft = 0;
 521       }
 522       $estimatedTime = ($recordsLeft / 50) * $time;
 523       $estMinutes = floor($estimatedTime / 60);
 524       $timeFormatted = '';
 525       if ($estMinutes > 1) {
 526         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 527         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 528       }
 529       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 530       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 531       $statusMsg = ts('%1 of %2 records - %3 remaining',
 532         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 533       );
 534       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 535       $contents = json_encode([$processedPercent, $status]);
 536
 537       file_put_contents($statusFile, $contents);
 538       return $currTimestamp;
 539     }
 540   }
 541
 542   /**
 543    * @return array
 544    */
 545   public function getSelectValues(): array {
 546     $values = [];
 547     foreach ($this->_fields as $name => $field) {
 548       $values[$name] = $field->_title;
 549     }
 550     return $values;
 551   }
 552
 553   /**
 554    * @return array
 555    */
 556   public function getSelectTypes() {
 557     $values = [];
 558     // This is only called from the MapField form in isolation now,
 559     // so we need to set the metadata.
 560     $this->init();
 561     foreach ($this->_fields as $name => $field) {
 562       if (isset($field->_hasLocationType)) {
 563         $values[$name] = $field->_hasLocationType;
 564       }
 565     }
 566     return $values;
 567   }
 568
 569   /**
 570    * @return array
 571    */
 572   public function getHeaderPatterns(): array {
 573     $values = [];
 574     foreach ($this->importableFieldsMetadata as $name => $field) {
 575       if (isset($field['headerPattern'])) {
 576         $values[$name] = $field['headerPattern'] ?: '//';
 577       }
 578     }
 579     return $values;
 580   }
 581
 582   /**
 583    * @return array
 584    */
 585   public function getDataPatterns():array {
 586     $values = [];
 587     foreach ($this->_fields as $name => $field) {
 588       $values[$name] = $field->_dataPattern;
 589     }
 590     return $values;
 591   }
 592
 593   /**
 594    * Remove single-quote enclosures from a value array (row).
 595    *
 596    * @param array $values
 597    * @param string $enclosure
 598    *
 599    * @return void
 600    */
 601   public static function encloseScrub(&$values, $enclosure = "'") {
 602     if (empty($values)) {
 603       return;
 604     }
 605
 606     foreach ($values as $k => $v) {
 607       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 608     }
 609   }
 610
 611   /**
 612    * Setter function.
 613    *
 614    * @param int $max
 615    *
 616    * @return void
 617    */
 618   public function setMaxLinesToProcess($max) {
 619     $this->_maxLinesToProcess = $max;
 620   }
 621
 622   /**
 623    * Validate that we have the required fields to create the contact or find it to update.
 624    *
 625    * Note that the users duplicate selection affects this as follows
 626    * - if they did not select an update variant then the id field is not
 627    *   permitted in the mapping - so we can assume the presence of id means
 628    *   we should use it
 629    * - the external_identifier field is valid in place of the other fields
 630    *   when they have chosen update or fill - in this case we are only looking
 631    *   to update an existing contact.
 632    *
 633    * @param string $contactType
 634    * @param array $params
 635    * @param bool $isPermitExistingMatchFields
 636    *   True if the it is enough to have fields which will enable us to find
 637    *   an existing contact (eg. external_identifier).
 638    * @param string $prefixString
 639    *   String to include in the exception (e.g '(Child of)' if we are validating
 640    *   a related contact.
 641    *
 642    * @return void
 643    * @throws \CRM_Core_Exception
 644    */
 645   protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void {
 646     if (!empty($params['id'])) {
 647       return;
 648     }
 649     $requiredFields = [
 650       'Individual' => [
 651         'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
 652         'email' => ts('Email Address'),
 653       ],
 654       'Organization' => ['organization_name' => ts('Organization Name')],
 655       'Household' => ['household_name' => ts('Household Name')],
 656     ][$contactType];
 657     if ($isPermitExistingMatchFields) {
 658       $requiredFields['external_identifier'] = ts('External Identifier');
 659       // Historically just an email has been accepted as it is 'usually good enough'
 660       // for a dedupe rule look up - but really this is a stand in for
 661       // whatever is needed to find an existing matching contact using the
 662       // specified dedupe rule (or the default Unsupervised if not specified).
 663       $requiredFields['email'] = ts('Email Address');
 664     }
 665     $this->validateRequiredFields($requiredFields, $params, $prefixString);
 666   }
 667
 668   protected function doPostImportActions() {
 669     $userJob = $this->getUserJob();
 670     $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
 671     $actions = $userJob['metadata']['post_actions'] ?? [];
 672     if (!empty($actions['group'])) {
 673       $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']);
 674       foreach ($actions['group'] as $groupID) {
 675         $summaryInfo['groups'][$groupID]['added'] += $groupAdditions[$groupID]['added'];
 676         $summaryInfo['groups'][$groupID]['notAdded'] += $groupAdditions[$groupID]['notAdded'];
 677       }
 678     }
 679     if (!empty($actions['tag'])) {
 680       $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts, $actions['tag']);
 681       foreach ($actions['tag'] as $tagID) {
 682         $summaryInfo['tags'][$tagID]['added'] += $tagAdditions[$tagID]['added'];
 683         $summaryInfo['tags'][$tagID]['notAdded'] += $tagAdditions[$tagID]['notAdded'];
 684       }
 685     }
 686
 687     $this->userJob['metadata']['summary_info'] = $summaryInfo;
 688     UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute();
 689   }
 690
 691   public function queue() {
 692     $dataSource = $this->getDataSourceObject();
 693     $totalRowCount = $totalRows = $dataSource->getRowCount(['new']);
 694     $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']);
 695     $offset = 0;
 696     $batchSize = 5;
 697     while ($totalRows > 0) {
 698       if ($totalRows < $batchSize) {
 699         $batchSize = $totalRows;
 700       }
 701       $task = new CRM_Queue_Task(
 702         [get_class($this), 'runImport'],
 703         ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize],
 704         ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount])
 705       );
 706       $queue->createItem($task);
 707       $totalRows -= $batchSize;
 708       $offset += $batchSize;
 709     }
 710
 711   }
 712
 713   /**
 714    * Add imported contacts to groups.
 715    *
 716    * @param array $contactIDs
 717    * @param array $groups
 718    *
 719    * @return array
 720    */
 721   private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array {
 722     $groupAdditions = [];
 723     foreach ($groups as $groupID) {
 724       // @todo - this function has been in use historically but it does not seem
 725       // to add much efficiency of get + create api calls
 726       // and it doesn't give enough control over cache flushing for smaller batches.
 727       // Note that the import updates a lot of enities & checking & updating the group
 728       // shouldn't add much performance wise. However, cache flushing will
 729       $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIDs, $groupID);
 730       $groupAdditions[$groupID] = [
 731         'added' => (int) $addCount[1],
 732         'notAdded' => (int) $addCount[2],
 733       ];
 734     }
 735     return $groupAdditions;
 736   }
 737
 738   /**
 739    * Tag imported contacts.
 740    *
 741    * @param array $contactIDs
 742    * @param array $tags
 743    *
 744    * @return array
 745    */
 746   private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) {
 747     $tagAdditions = [];
 748     foreach ($tags as $tagID) {
 749       // @todo - this function has been in use historically but it does not seem
 750       // to add much efficiency of get + create api calls
 751       // and it doesn't give enough control over cache flushing for smaller batches.
 752       // Note that the import updates a lot of enities & checking & updating the group
 753       // shouldn't add much performance wise. However, cache flushing will
 754       $outcome = CRM_Core_BAO_EntityTag::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE);
 755       $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]];
 756     }
 757     return $tagAdditions;
 758   }
 759
 760   /**
 761    * Determines the file extension based on error code.
 762    *
 763    * @var int $type error code constant
 764    * @return string
 765    */
 766   public static function errorFileName($type) {
 767     $fileName = NULL;
 768     if (empty($type)) {
 769       return $fileName;
 770     }
 771
 772     $config = CRM_Core_Config::singleton();
 773     $fileName = $config->uploadDir . "sqlImport";
 774     switch ($type) {
 775       case self::ERROR:
 776         $fileName .= '.errors';
 777         break;
 778
 779       case self::DUPLICATE:
 780         $fileName .= '.duplicates';
 781         break;
 782
 783       case self::NO_MATCH:
 784         $fileName .= '.mismatch';
 785         break;
 786
 787       case self::UNPARSED_ADDRESS_WARNING:
 788         $fileName .= '.unparsedAddress';
 789         break;
 790     }
 791
 792     return $fileName;
 793   }
 794
 795   /**
 796    * Determines the file name based on error code.
 797    *
 798    * @var $type error code constant
 799    * @return string
 800    */
 801   public static function saveFileName($type) {
 802     $fileName = NULL;
 803     if (empty($type)) {
 804       return $fileName;
 805     }
 806     switch ($type) {
 807       case self::ERROR:
 808         $fileName = 'Import_Errors.csv';
 809         break;
 810
 811       case self::DUPLICATE:
 812         $fileName = 'Import_Duplicates.csv';
 813         break;
 814
 815       case self::NO_MATCH:
 816         $fileName = 'Import_Mismatch.csv';
 817         break;
 818
 819       case self::UNPARSED_ADDRESS_WARNING:
 820         $fileName = 'Import_Unparsed_Address.csv';
 821         break;
 822     }
 823
 824     return $fileName;
 825   }
 826
 827   /**
 828    * Check if contact is a duplicate .
 829    *
 830    * @param array $formatValues
 831    *
 832    * @return array
 833    */
 834   protected function checkContactDuplicate(&$formatValues) {
 835     //retrieve contact id using contact dedupe rule
 836     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->getContactType();
 837     $formatValues['version'] = 3;
 838     require_once 'CRM/Utils/DeprecatedUtils.php';
 839     $params = $formatValues;
 840     static $cIndieFields = NULL;
 841     static $defaultLocationId = NULL;
 842
 843     $contactType = $params['contact_type'];
 844     if ($cIndieFields == NULL) {
 845       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 846       $cIndieFields = $cTempIndieFields;
 847
 848       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 849
 850       // set the value to default location id else set to 1
 851       if (!$defaultLocationId = (int) $defaultLocation->id) {
 852         $defaultLocationId = 1;
 853       }
 854     }
 855
 856     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 857
 858     $contactFormatted = [];
 859     foreach ($params as $key => $field) {
 860       if ($field == NULL || $field === '') {
 861         continue;
 862       }
 863       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 864       // instead of soft credit contact.
 865       if (is_array($field) && $key !== "soft_credit") {
 866         foreach ($field as $value) {
 867           $break = FALSE;
 868           if (is_array($value)) {
 869             foreach ($value as $name => $testForEmpty) {
 870               if ($name !== 'phone_type' &&
 871                 ($testForEmpty === '' || $testForEmpty == NULL)
 872               ) {
 873                 $break = TRUE;
 874                 break;
 875               }
 876             }
 877           }
 878           else {
 879             $break = TRUE;
 880           }
 881           if (!$break) {
 882             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 883           }
 884         }
 885         continue;
 886       }
 887
 888       $value = [$key => $field];
 889
 890       // check if location related field, then we need to add primary location type
 891       if (in_array($key, $locationFields)) {
 892         $value['location_type_id'] = $defaultLocationId;
 893       }
 894       elseif (array_key_exists($key, $cIndieFields)) {
 895         $value['contact_type'] = $contactType;
 896       }
 897
 898       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 899     }
 900
 901     $contactFormatted['contact_type'] = $contactType;
 902
 903     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 904   }
 905
 906   /**
 907    * This function adds the contact variable in $values to the
 908    * parameter list $params.  For most cases, $values should have length 1.  If
 909    * the variable being added is a child of Location, a location_type_id must
 910    * also be included.  If it is a child of phone, a phone_type must be included.
 911    *
 912    * @param array $values
 913    *   The variable(s) to be added.
 914    * @param array $params
 915    *   The structured parameter list.
 916    *
 917    * @return bool|CRM_Utils_Error
 918    */
 919   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 920     // @todo - like most functions in import ... most of this is cruft....
 921     // Crawl through the possible classes:
 922     // Contact
 923     //      Individual
 924     //      Household
 925     //      Organization
 926     //          Location
 927     //              Address
 928     //              Email
 929     //              Phone
 930     //              IM
 931     //      Note
 932     //      Custom
 933
 934     // Cache the various object fields
 935     static $fields = NULL;
 936
 937     if ($fields == NULL) {
 938       $fields = [];
 939     }
 940
 941     // first add core contact values since for other Civi modules they are not added
 942     require_once 'CRM/Contact/BAO/Contact.php';
 943     $contactFields = CRM_Contact_DAO_Contact::fields();
 944     _civicrm_api3_store_values($contactFields, $values, $params);
 945
 946     if (isset($values['contact_type'])) {
 947       // we're an individual/household/org property
 948
 949       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 950
 951       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 952       return TRUE;
 953     }
 954
 955     if (isset($values['individual_prefix'])) {
 956       if (!empty($params['prefix_id'])) {
 957         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 958         $params['prefix'] = $prefixes[$params['prefix_id']];
 959       }
 960       else {
 961         $params['prefix'] = $values['individual_prefix'];
 962       }
 963       return TRUE;
 964     }
 965
 966     if (isset($values['individual_suffix'])) {
 967       if (!empty($params['suffix_id'])) {
 968         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 969         $params['suffix'] = $suffixes[$params['suffix_id']];
 970       }
 971       else {
 972         $params['suffix'] = $values['individual_suffix'];
 973       }
 974       return TRUE;
 975     }
 976
 977     if (isset($values['gender'])) {
 978       if (!empty($params['gender_id'])) {
 979         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 980         $params['gender'] = $genders[$params['gender_id']];
 981       }
 982       else {
 983         $params['gender'] = $values['gender'];
 984       }
 985       return TRUE;
 986     }
 987
 988     // format the website params.
 989     if (!empty($values['url'])) {
 990       static $websiteFields;
 991       if (!is_array($websiteFields)) {
 992         require_once 'CRM/Core/DAO/Website.php';
 993         $websiteFields = CRM_Core_DAO_Website::fields();
 994       }
 995       if (!array_key_exists('website', $params) ||
 996         !is_array($params['website'])
 997       ) {
 998         $params['website'] = [];
 999       }
1000
1001       $websiteCount = count($params['website']);
1002       _civicrm_api3_store_values($websiteFields, $values,
1003         $params['website'][++$websiteCount]
1004       );
1005
1006       return TRUE;
1007     }
1008
1009     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1010     if (!empty($values['location_type_id'])) {
1011       static $fields = NULL;
1012       if ($fields == NULL) {
1013         $fields = [];
1014       }
1015
1016       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
1017         $name = strtolower($block);
1018         if (!array_key_exists($name, $values)) {
1019           continue;
1020         }
1021
1022         if ($name === 'phone_ext') {
1023           $block = 'Phone';
1024         }
1025
1026         // block present in value array.
1027         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
1028           $params[$name] = [];
1029         }
1030
1031         if (!array_key_exists($block, $fields)) {
1032           $className = "CRM_Core_DAO_$block";
1033           $fields[$block] =& $className::fields();
1034         }
1035
1036         $blockCnt = count($params[$name]);
1037
1038         // copy value to dao field name.
1039         if ($name == 'im') {
1040           $values['name'] = $values[$name];
1041         }
1042
1043         _civicrm_api3_store_values($fields[$block], $values,
1044           $params[$name][++$blockCnt]
1045         );
1046
1047         if (empty($params['id']) && ($blockCnt == 1)) {
1048           $params[$name][$blockCnt]['is_primary'] = TRUE;
1049         }
1050
1051         // we only process single block at a time.
1052         return TRUE;
1053       }
1054
1055       // handle address fields.
1056       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1057         $params['address'] = [];
1058       }
1059
1060       $addressCnt = 1;
1061       foreach ($params['address'] as $cnt => $addressBlock) {
1062         if (CRM_Utils_Array::value('location_type_id', $values) ==
1063           CRM_Utils_Array::value('location_type_id', $addressBlock)
1064         ) {
1065           $addressCnt = $cnt;
1066           break;
1067         }
1068         $addressCnt++;
1069       }
1070
1071       if (!array_key_exists('Address', $fields)) {
1072         $fields['Address'] = CRM_Core_DAO_Address::fields();
1073       }
1074
1075       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1076       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1077       // the address in CRM_Core_BAO_Address::create method
1078       if (!empty($values['location_type_id'])) {
1079         static $customFields = [];
1080         if (empty($customFields)) {
1081           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1082         }
1083         // make a copy of values, as we going to make changes
1084         $newValues = $values;
1085         foreach ($values as $key => $val) {
1086           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1087           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1088             // mark an entry in fields array since we want the value of custom field to be copied
1089             $fields['Address'][$key] = NULL;
1090
1091             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1092             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1093               $mulValues = explode(',', $val);
1094               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1095               $newValues[$key] = [];
1096               foreach ($mulValues as $v1) {
1097                 foreach ($customOption as $v2) {
1098                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1099                     (strtolower($v2['value']) == strtolower(trim($v1)))
1100                   ) {
1101                     if ($htmlType == 'CheckBox') {
1102                       $newValues[$key][$v2['value']] = 1;
1103                     }
1104                     else {
1105                       $newValues[$key][] = $v2['value'];
1106                     }
1107                   }
1108                 }
1109               }
1110             }
1111           }
1112         }
1113         // consider new values
1114         $values = $newValues;
1115       }
1116
1117       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1118
1119       $addressFields = [
1120         'county',
1121         'country',
1122         'state_province',
1123         'supplemental_address_1',
1124         'supplemental_address_2',
1125         'supplemental_address_3',
1126         'StateProvince.name',
1127       ];
1128
1129       foreach ($addressFields as $field) {
1130         if (array_key_exists($field, $values)) {
1131           if (!array_key_exists('address', $params)) {
1132             $params['address'] = [];
1133           }
1134           $params['address'][$addressCnt][$field] = $values[$field];
1135         }
1136       }
1137
1138       if ($addressCnt == 1) {
1139
1140         $params['address'][$addressCnt]['is_primary'] = TRUE;
1141       }
1142       return TRUE;
1143     }
1144
1145     if (isset($values['note'])) {
1146       // add a note field
1147       if (!isset($params['note'])) {
1148         $params['note'] = [];
1149       }
1150       $noteBlock = count($params['note']) + 1;
1151
1152       $params['note'][$noteBlock] = [];
1153       if (!isset($fields['Note'])) {
1154         $fields['Note'] = CRM_Core_DAO_Note::fields();
1155       }
1156
1157       // get the current logged in civicrm user
1158       $session = CRM_Core_Session::singleton();
1159       $userID = $session->get('userID');
1160
1161       if ($userID) {
1162         $values['contact_id'] = $userID;
1163       }
1164
1165       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1166
1167       return TRUE;
1168     }
1169
1170     // Check for custom field values
1171
1172     if (empty($fields['custom'])) {
1173       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1174         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1175       );
1176     }
1177
1178     foreach ($values as $key => $value) {
1179       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1180         // check if it's a valid custom field id
1181
1182         if (!array_key_exists($customFieldID, $fields['custom'])) {
1183           return civicrm_api3_create_error('Invalid custom field ID');
1184         }
1185         else {
1186           $params[$key] = $value;
1187         }
1188       }
1189     }
1190   }
1191
1192   /**
1193    * Parse a field which could be represented by a label or name value rather than the DB value.
1194    *
1195    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1196    *
1197    * but if not available then see if we have a label that can be converted to a name.
1198    *
1199    * @param string|int|null $submittedValue
1200    * @param array $fieldSpec
1201    *   Metadata for the field
1202    *
1203    * @return mixed
1204    */
1205   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1206     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1207     if (!isset($fieldSpec['bao'])) {
1208       return $submittedValue;
1209     }
1210     /* @var \CRM_Core_DAO $bao */
1211     $bao = $fieldSpec['bao'];
1212     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1213     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1214     if (isset($nameOptions[$submittedValue])) {
1215       return $submittedValue;
1216     }
1217     if (in_array($submittedValue, $nameOptions)) {
1218       return array_search($submittedValue, $nameOptions, TRUE);
1219     }
1220
1221     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1222     if (isset($labelOptions[$submittedValue])) {
1223       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1224     }
1225     return '';
1226   }
1227
1228   /**
1229    * This is code extracted from 4 places where this exact snippet was being duplicated.
1230    *
1231    * FIXME: Extracting this was a first step, but there's also
1232    *  1. Inconsistency in the way other select options are handled.
1233    *     Contribution adds handling for Select/Radio/Autocomplete
1234    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1235    *     Membership is missing all of it
1236    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1237    *
1238    * @param $customFieldID
1239    * @param $value
1240    * @param $fieldType
1241    * @return array
1242    */
1243   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1244     $mulValues = explode(',', $value);
1245     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1246     $values = [];
1247     foreach ($mulValues as $v1) {
1248       foreach ($customOption as $customValueID => $customLabel) {
1249         $customValue = $customLabel['value'];
1250         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1251           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1252         ) {
1253           $values[] = $customValue;
1254         }
1255       }
1256     }
1257     return $values;
1258   }
1259
1260   /**
1261    * Validate that the field requirements are met in the params.
1262    *
1263    * @param array $requiredFields
1264    * @param array $params
1265    *   An array of required fields (fieldName => label)
1266    *   - note this follows the and / or array nesting we see in permission checks
1267    *   eg.
1268    *   [
1269    *     'email' => ts('Email'),
1270    *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1271    *   ]
1272    *   Means 'email' OR 'first_name AND 'last_name'.
1273    * @param string $prefixString
1274    *
1275    * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1276    */
1277   protected function validateRequiredFields(array $requiredFields, array $params, $prefixString = ''): void {
1278     $missingFields = [];
1279     foreach ($requiredFields as $key => $required) {
1280       if (!is_array($required)) {
1281         $importParameter = $params[$key] ?? [];
1282         if (!is_array($importParameter)) {
1283           if (!empty($importParameter)) {
1284             return;
1285           }
1286         }
1287         else {
1288           foreach ($importParameter as $locationValues) {
1289             if (!empty($locationValues[$key])) {
1290               return;
1291             }
1292           }
1293         }
1294
1295         $missingFields[$key] = $required;
1296       }
1297       else {
1298         foreach ($required as $field => $label) {
1299           if (empty($params[$field])) {
1300             $missing[$field] = $label;
1301           }
1302         }
1303         if (empty($missing)) {
1304           return;
1305         }
1306         $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1307       }
1308     }
1309     throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1310   }
1311
1312   /**
1313    * Get the field value, transformed by metadata.
1314    *
1315    * @param string $fieldName
1316    * @param string|int $importedValue
1317    *   Value as it came in from the datasource.
1318    *
1319    * @return string|array|bool|int
1320    * @throws \API_Exception
1321    */
1322   protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1323     if (empty($importedValue)) {
1324       return $importedValue;
1325     }
1326     $fieldMetadata = $this->getFieldMetadata($fieldName);
1327     if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) {
1328       $values = [];
1329       foreach (explode(',', $importedValue) as $value) {
1330         $values[] = $this->getTransformedFieldValue($fieldName, trim($value));
1331       }
1332       return $values;
1333     }
1334     if ($fieldName === 'url') {
1335       return CRM_Utils_Rule::url($importedValue) ? $importedValue : 'invalid_import_value';
1336     }
1337
1338     if ($fieldName === 'email') {
1339       return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value';
1340     }
1341
1342     if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) {
1343       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1344     }
1345     if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) {
1346       return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value';
1347     }
1348     if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) {
1349       $value = CRM_Utils_String::strtoboolstr($importedValue);
1350       if ($value !== FALSE) {
1351         return (bool) $value;
1352       }
1353       return 'invalid_import_value';
1354     }
1355     if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) {
1356       $value = CRM_Utils_Date::formatDate($importedValue, $this->getSubmittedValue('dateFormats'));
1357       return ($value) ?: 'invalid_import_value';
1358     }
1359     $options = $this->getFieldOptions($fieldName);
1360     if ($options !== FALSE) {
1361       if ($this->isAmbiguous($fieldName, $importedValue)) {
1362         // We can't transform it at this stage. Perhaps later we can with
1363         // other information such as country.
1364         return $importedValue;
1365       }
1366
1367       $comparisonValue = $this->getComparisonValue($importedValue);
1368       return $options[$comparisonValue] ?? 'invalid_import_value';
1369     }
1370     if (!empty($fieldMetadata['FKClassName']) || !empty($fieldMetadata['pseudoconstant']['prefetch'])) {
1371       // @todo - make this generic - for fields where getOptions doesn't fetch
1372       // getOptions does not retrieve these fields with high potential results
1373       if ($fieldName === 'event_id') {
1374         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1375           $event = Event::get()->addClause('OR', ['title', '=', $importedValue], ['id', '=', $importedValue])->addSelect('id')->execute()->first();
1376           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $event['id'] ?? FALSE;
1377         }
1378         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1379       }
1380       if ($fieldMetadata['name'] === 'campaign_id') {
1381         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1382           $campaign = Campaign::get()->addClause('OR', ['title', '=', $importedValue], ['name', '=', $importedValue])->addSelect('id')->execute()->first();
1383           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $campaign['id'] ?? FALSE;
1384         }
1385         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1386       }
1387     }
1388     return $importedValue;
1389   }
1390
1391   /**
1392    * @param string $fieldName
1393    *
1394    * @return false|array
1395    *
1396    * @throws \API_Exception
1397    */
1398   protected function getFieldOptions(string $fieldName) {
1399     return $this->getFieldMetadata($fieldName, TRUE)['options'];
1400   }
1401
1402   /**
1403    * Get the metadata for the field.
1404    *
1405    * @param string $fieldName
1406    * @param bool $loadOptions
1407    * @param bool $limitToContactType
1408    *   Only show fields for the type to import (not appropriate when looking up
1409    *   related contact fields).
1410    *
1411    * @return array
1412    *
1413    * @noinspection PhpDocMissingThrowsInspection
1414    * @noinspection PhpUnhandledExceptionInspection
1415    */
1416   protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1417
1418     $fieldMap = $this->getOddlyMappedMetadataFields();
1419     $fieldMapName = empty($fieldMap[$fieldName]) ? $fieldName : $fieldMap[$fieldName];
1420
1421     // This whole business of only loading metadata for one type when we actually need it for all is ... dubious.
1422     if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) {
1423       if ($loadOptions || !$limitToContactType) {
1424         $this->importableFieldsMetadata[$fieldMapName] = CRM_Contact_BAO_Contact::importableFields('All')[$fieldMapName];
1425       }
1426     }
1427
1428     $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName];
1429     if ($loadOptions && !isset($fieldMetadata['options'])) {
1430       if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') {
1431         // Probably already loaded and also supports abbreviations - eg. NSW.
1432         // Supporting for core AND custom state fields is more consistent.
1433         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id');
1434         return $this->importableFieldsMetadata[$fieldMapName];
1435       }
1436       if (($fieldMetadata['data_type'] ?? '') === 'Country') {
1437         // Probably already loaded and also supports abbreviations - eg. NSW.
1438         // Supporting for core AND custom state fields is more consistent.
1439         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('country_id');
1440         return $this->importableFieldsMetadata[$fieldMapName];
1441       }
1442       $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;
1443
1444       if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
1445         $options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
1446           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1447           'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
1448           'select' => ['options'],
1449         ])->first()['options'];
1450       }
1451       else {
1452         if (!empty($fieldMetadata['custom_group_id'])) {
1453           $customField = CustomField::get(FALSE)
1454             ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
1455             ->addSelect('name', 'custom_group_id.name')
1456             ->execute()
1457             ->first();
1458           $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
1459         }
1460         $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1461           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1462           'where' => [['name', '=', $optionFieldName]],
1463           'select' => ['options'],
1464         ])->first()['options'];
1465       }
1466       if (is_array($options)) {
1467         // We create an array of the possible variants - notably including
1468         // name AND label as either might be used. We also lower case before checking
1469         $values = [];
1470         foreach ($options as $option) {
1471           $idKey = $this->getComparisonValue($option['id']);
1472           $values[$idKey] = $option['id'];
1473           foreach (['name', 'label', 'abbr'] as $key) {
1474             $optionValue = $this->getComparisonValue($option[$key] ?? '');
1475             if ($optionValue !== '') {
1476               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
1477                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
1478                   $this->ambiguousOptions[$fieldName][$optionValue] = [$values[$optionValue]];
1479                 }
1480                 $this->ambiguousOptions[$fieldName][$optionValue][] = $option['id'];
1481               }
1482               else {
1483                 $values[$optionValue] = $option['id'];
1484               }
1485             }
1486           }
1487         }
1488         $this->importableFieldsMetadata[$fieldMapName]['options'] = $values;
1489       }
1490       else {
1491         $this->importableFieldsMetadata[$fieldMapName]['options'] = $options ?: FALSE;
1492       }
1493       return $this->importableFieldsMetadata[$fieldMapName];
1494     }
1495     return $fieldMetadata;
1496   }
1497
1498   /**
1499    * Get the field metadata for fields to be be offered to match the contact.
1500    *
1501    * @return array
1502    * @noinspection PhpDocMissingThrowsInspection
1503    */
1504   protected function getContactMatchingFields(): array {
1505     $contactFields = CRM_Contact_BAO_Contact::importableFields($this->getContactType(), NULL);
1506     $fields = ['external_identifier' => $contactFields['external_identifier']];
1507     $fields['external_identifier']['title'] .= ' (match to contact)';
1508     // Using new Dedupe rule.
1509     $ruleParams = [
1510       'contact_type' => $this->getContactType(),
1511       'used' => $this->getSubmittedValue('dedupe_rule_id') ?? 'Unsupervised',
1512     ];
1513     $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
1514
1515     if (is_array($fieldsArray)) {
1516       foreach ($fieldsArray as $value) {
1517         $customFieldId = CRM_Core_DAO::getFieldValue('CRM_Core_DAO_CustomField',
1518           $value,
1519           'id',
1520           'column_name'
1521         );
1522         $value = trim($customFieldId ? 'custom_' . $customFieldId : $value);
1523         $fields[$value] = $contactFields[$value] ?? NULL;
1524         $title = $fields[$value]['title'] . ' (match to contact)';
1525         $fields[$value]['title'] = $title;
1526       }
1527     }
1528     return $fields;
1529   }
1530
1531   /**
1532    * @param $customFieldID
1533    * @param $value
1534    * @param array $fieldMetaData
1535    * @param $dateType
1536    *
1537    * @return ?string
1538    */
1539   protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?string {
1540     /* validate the data against the CF type */
1541
1542     if ($value) {
1543       $dataType = $fieldMetaData['data_type'];
1544       $htmlType = $fieldMetaData['html_type'];
1545       $isSerialized = CRM_Core_BAO_CustomField::isSerialized($fieldMetaData);
1546       if ($dataType === 'Date') {
1547         $params = ['date_field' => $value];
1548         if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, 'date_field')) {
1549           return NULL;
1550         }
1551         return $fieldMetaData['label'];
1552       }
1553       elseif ($dataType === 'Boolean') {
1554         if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1555           return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1556         }
1557       }
1558       // need not check for label filed import
1559       $selectHtmlTypes = [
1560         'CheckBox',
1561         'Select',
1562         'Radio',
1563       ];
1564       if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1565         $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1566         if (!$valid) {
1567           return $fieldMetaData['label'];
1568         }
1569       }
1570
1571       // check for values for custom fields for checkboxes and multiselect
1572       if ($isSerialized && $dataType != 'ContactReference') {
1573         $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1574         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1575         foreach ($mulValues as $v1) {
1576
1577           $flag = FALSE;
1578           foreach ($customOption as $v2) {
1579             if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1580               $flag = TRUE;
1581             }
1582           }
1583
1584           if (!$flag) {
1585             return $fieldMetaData['label'];
1586           }
1587         }
1588       }
1589       elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1590         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1591         $flag = FALSE;
1592         foreach ($customOption as $v2) {
1593           if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1594             $flag = TRUE;
1595           }
1596         }
1597         if (!$flag) {
1598           return $fieldMetaData['label'];
1599         }
1600       }
1601     }
1602
1603     return NULL;
1604   }
1605
1606   /**
1607    * Get the entity for the given field.
1608    *
1609    * @param string $fieldName
1610    *
1611    * @return mixed|null
1612    * @throws \API_Exception
1613    */
1614   protected function getFieldEntity(string $fieldName) {
1615     if ($fieldName === 'do_not_import') {
1616       return NULL;
1617     }
1618     if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) {
1619       return 'Contact';
1620     }
1621     $metadata = $this->getFieldMetadata($fieldName);
1622     if (!isset($metadata['entity'])) {
1623       return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ? 'Contact' : $metadata['extends'];
1624     }
1625
1626     // Our metadata for these is fugly. Handling the fugliness during retrieval.
1627     if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1628       return 'Address';
1629     }
1630     return $metadata['entity'];
1631   }
1632
1633   /**
1634    * Validate the import file, updating the import table with results.
1635    *
1636    * @throws \API_Exception
1637    * @throws \CRM_Core_Exception
1638    */
1639   public function validate(): void {
1640     $dataSource = $this->getDataSourceObject();
1641     while ($row = $dataSource->getRow()) {
1642       try {
1643         $rowNumber = $row['_id'];
1644         $values = array_values($row);
1645         $this->validateValues($values);
1646         $this->setImportStatus($rowNumber, 'NEW', '');
1647       }
1648       catch (CRM_Core_Exception $e) {
1649         $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
1650       }
1651     }
1652   }
1653
1654   /**
1655    * Validate the import values.
1656    *
1657    * The values array represents a row in the datasource.
1658    *
1659    * @param array $values
1660    *
1661    * @throws \API_Exception
1662    * @throws \CRM_Core_Exception
1663    */
1664   public function validateValues(array $values): void {
1665     $params = $this->getMappedRow($values);
1666     $this->validateParams($params);
1667   }
1668
1669   /**
1670    * @param array $params
1671    *
1672    * @throws \CRM_Core_Exception
1673    */
1674   protected function validateParams(array $params): void {
1675     if (empty($params['id'])) {
1676       $this->validateRequiredFields($this->getRequiredFields(), $params);
1677     }
1678     $errors = [];
1679     foreach ($params as $key => $value) {
1680       $errors = array_merge($this->getInvalidValues($value, $key), $errors);
1681     }
1682     if ($errors) {
1683       throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors));
1684     }
1685   }
1686
1687   /**
1688    * Search the value for the string 'invalid_import_value'.
1689    *
1690    * If the string is found it indicates the fields was rejected
1691    * during `getTransformedValue` as not having valid data.
1692    *
1693    * @param string|array|int $value
1694    * @param string $key
1695    * @param string $prefixString
1696    *
1697    * @return array
1698    */
1699   protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
1700     $errors = [];
1701     if ($value === 'invalid_import_value') {
1702       if (!is_numeric($key)) {
1703         $metadata = $this->getFieldMetadata($key);
1704         $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1705       }
1706       else {
1707         // Numeric key suggests we are drilling into option values
1708         $errors[] = TRUE;
1709       }
1710     }
1711     elseif (is_array($value)) {
1712       foreach ($value as $innerKey => $innerValue) {
1713         $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1714         if ($result === [TRUE]) {
1715           $metadata = $this->getFieldMetadata($key);
1716           $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1717         }
1718         elseif (!empty($result)) {
1719           $errors = array_merge($result, $errors);
1720         }
1721       }
1722     }
1723     return array_filter($errors);
1724   }
1725
1726   /**
1727    * Get the available countries.
1728    *
1729    * If the site is not configured with a restriction then all countries are valid
1730    * but otherwise only a select array are.
1731    *
1732    * @return array|false
1733    *   FALSE indicates no restrictions.
1734    */
1735   protected function getAvailableCountries() {
1736     if ($this->availableCountries === NULL) {
1737       $availableCountries = Civi::settings()->get('countryLimit');
1738       $this->availableCountries = !empty($availableCountries) ? array_fill_keys($availableCountries, TRUE) : FALSE;
1739     }
1740     return $this->availableCountries;
1741   }
1742
1743   /**
1744    * Get the metadata field for which importable fields does not key the actual field name.
1745    *
1746    * @return string[]
1747    */
1748   protected function getOddlyMappedMetadataFields(): array {
1749     return [
1750       'country_id' => 'country',
1751       'state_province_id' => 'state_province',
1752       'county_id' => 'county',
1753       'email_greeting_id' => 'email_greeting',
1754       'postal_greeting_id' => 'postal_greeting',
1755       'addressee_id' => 'addressee',
1756     ];
1757   }
1758
1759   /**
1760    * Get the default country for the site.
1761    *
1762    * @return int
1763    */
1764   protected function getSiteDefaultCountry(): int {
1765     if (!isset($this->siteDefaultCountry)) {
1766       $this->siteDefaultCountry = (int) Civi::settings()->get('defaultContactCountry');
1767     }
1768     return $this->siteDefaultCountry;
1769   }
1770
1771   /**
1772    * Is the option ambiguous.
1773    *
1774    * @param string $fieldName
1775    * @param string $importedValue
1776    */
1777   protected function isAmbiguous(string $fieldName, $importedValue): bool {
1778     return !empty($this->ambiguousOptions[$fieldName][$this->getComparisonValue($importedValue)]);
1779   }
1780
1781   /**
1782    * Get the civicrm_mapping_field appropriate layout for the mapper input.
1783    *
1784    * For simple parsers (not contribution or contact) the input looks like
1785    * ['first_name', 'custom_32']
1786    * and it is converted to
1787    *
1788    *  ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5],
1789    *
1790    * @param array $fieldMapping
1791    * @param int $mappingID
1792    * @param int $columnNumber
1793    *
1794    * @return array
1795    */
1796   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
1797     return [
1798       'name' => $fieldMapping[0],
1799       'mapping_id' => $mappingID,
1800       'column_number' => $columnNumber,
1801     ];
1802   }
1803
1804   /**
1805    * The initializer code, called before the processing
1806    *
1807    * @return void
1808    */
1809   public function init() {
1810     // Force re-load of user job.
1811     unset($this->userJob);
1812     $this->setFieldMetadata();
1813   }
1814
1815   /**
1816    * @param array $mappedField
1817    *   Field detail as would be saved in field_mapping table
1818    *   or as returned from getMappingFieldFromMapperInput
1819    *
1820    * @return string
1821    * @throws \API_Exception
1822    */
1823   public function getMappedFieldLabel(array $mappedField): string {
1824     // doNotImport is on it's way out - skip fields will be '' once all is done.
1825     if ($mappedField['name'] === 'doNotImport') {
1826       return '';
1827     }
1828     $this->setFieldMetadata();
1829     $metadata = $this->getFieldMetadata($mappedField['name']);
1830     return $metadata['html']['label'] ?? $metadata['title'];
1831   }
1832
1833   /**
1834    * Get the row from the csv mapped to our parameters.
1835    *
1836    * @param array $values
1837    *
1838    * @return array
1839    * @throws \API_Exception
1840    */
1841   public function getMappedRow(array $values): array {
1842     $params = [];
1843     foreach ($this->getFieldMappings() as $i => $mappedField) {
1844       if ($mappedField['name'] === 'do_not_import') {
1845         continue;
1846       }
1847       if ($mappedField['name']) {
1848         $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
1849       }
1850     }
1851     return $params;
1852   }
1853
1854   /**
1855    * Get the field mappings for the import.
1856    *
1857    * This is the same format as saved in civicrm_mapping_field except
1858    * that location_type_id = 'Primary' rather than empty where relevant.
1859    * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
1860    *
1861    * @return array
1862    * @throws \API_Exception
1863    */
1864   protected function getFieldMappings(): array {
1865     $mappedFields = [];
1866     $mapper = $this->getSubmittedValue('mapper');
1867     foreach ($mapper as $i => $mapperRow) {
1868       $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
1869       // Just for clarity since 0 is a pseudo-value
1870       unset($mappedField['mapping_id']);
1871       $mappedFields[] = $mappedField;
1872     }
1873     return $mappedFields;
1874   }
1875
1876   /**
1877    * Run import.
1878    *
1879    * @param \CRM_Queue_TaskContext $taskContext
1880    *
1881    * @param int $userJobID
1882    * @param int $limit
1883    *
1884    * @return bool
1885    * @throws \API_Exception
1886    * @throws \CRM_Core_Exception
1887    */
1888   public static function runImport($taskContext, $userJobID, $limit) {
1889     $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('type_id')->execute()->first();
1890     $parserClass = NULL;
1891     foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
1892       if ($userJob['type_id'] === $userJobType['id']) {
1893         $parserClass = $userJobType['class'];
1894       }
1895     }
1896     /* @var \CRM_Import_Parser $parser */
1897     $parser = new $parserClass();
1898     $parser->setUserJobID($userJobID);
1899     // Not sure if we still need to init....
1900     $parser->init();
1901     $dataSource = $parser->getDataSourceObject();
1902     $dataSource->setStatuses(['new']);
1903     $dataSource->setLimit($limit);
1904
1905     while ($row = $dataSource->getRow()) {
1906       $values = array_values($row);
1907       $parser->import($values);
1908     }
1909     $parser->doPostImportActions();
1910     return TRUE;
1911   }
1912
1913   /**
1914    * Check if an error in custom data.
1915    *
1916    * @deprecated all of this is duplicated if getTransformedValue is used.
1917    *
1918    * @param array $params
1919    * @param string $errorMessage
1920    *   A string containing all the error-fields.
1921    *
1922    * @param null $csType
1923    */
1924   public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) {
1925     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1926     $errors = [];
1927
1928     if (!empty($params['contact_sub_type'])) {
1929       $csType = $params['contact_sub_type'] ?? NULL;
1930     }
1931
1932     if (empty($params['contact_type'])) {
1933       $params['contact_type'] = 'Individual';
1934     }
1935
1936     // get array of subtypes - CRM-18708
1937     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1938       $csType = $this->getSubtypes($params['contact_type']);
1939     }
1940
1941     if (is_array($csType)) {
1942       // fetch custom fields for every subtype and add it to $customFields array
1943       // CRM-18708
1944       $customFields = [];
1945       foreach ($csType as $cType) {
1946         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1947       }
1948     }
1949     else {
1950       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1951     }
1952
1953     foreach ($params as $key => $value) {
1954       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1955         //For address custom fields, we do get actual custom field value as an inner array of
1956         //values so need to modify
1957         if (!array_key_exists($customFieldID, $customFields)) {
1958           return ts('field ID');
1959         }
1960         /* check if it's a valid custom field id */
1961         $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType);
1962       }
1963     }
1964     if ($errors) {
1965       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', array_filter($errors));
1966     }
1967   }
1968
1969   /**
1970    * get subtypes given the contact type
1971    *
1972    * @param string $contactType
1973    * @return array $subTypes
1974    */
1975   protected function getSubtypes($contactType) {
1976     $subTypes = [];
1977     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
1978
1979     if (count($types) > 0) {
1980       foreach ($types as $type) {
1981         $subTypes[] = $type['name'];
1982       }
1983     }
1984     return $subTypes;
1985   }
1986
1987   /**
1988    * Update the status of the import row to reflect the processing outcome.
1989    *
1990    * @param int $id
1991    * @param string $status
1992    * @param string $message
1993    * @param int|null $entityID
1994    *   Optional created entity ID
1995    * @param array $additionalFields
1996    *  Additional fields to be tracked
1997    * @param array $createdContactIDs
1998    *
1999    * @noinspection PhpDocMissingThrowsInspection
2000    * @noinspection PhpUnhandledExceptionInspection
2001    */
2002   protected function setImportStatus(int $id, string $status, string $message = '', ?int $entityID = NULL, $additionalFields = [], $createdContactIDs = []): void {
2003     foreach ($createdContactIDs as $createdContactID) {
2004       // Store any created contacts for post_actions like tag or add to group.
2005       // These are done on a 'per-batch' status in processPorstActions
2006       // so holding in a property is OK.
2007       $this->createdContacts[$createdContactID] = $createdContactID;
2008     }
2009     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields);
2010   }
2011
2012   /**
2013    * Convert any given date string to default date array.
2014    *
2015    * @param array $params
2016    *   Has given date-format.
2017    * @param array $formatted
2018    *   Store formatted date in this array.
2019    * @param int $dateType
2020    *   Type of date.
2021    * @param string $dateParam
2022    *   Index of params.
2023    */
2024   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2025     //fix for CRM-2687
2026     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
2027     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
2028   }
2029
2030   /**
2031    * Get the value to use for option comparison purposes.
2032    *
2033    * We do a case-insensitive comparison, also swapping ’ for '
2034    * which has at least one known usage (Côte d’Ivoire).
2035    *
2036    * Note we do this to both sides of the comparison.
2037    *
2038    * @param int|string|false|null $importedValue
2039    *
2040    * @return false|int|string|null
2041    */
2042   protected function getComparisonValue($importedValue) {
2043     return is_numeric($importedValue) ? $importedValue : mb_strtolower(str_replace('’', "'", $importedValue));
2044   }
2045
2046 }