CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Campaign;
  13 use Civi\Api4\CustomField;
  14 use Civi\Api4\Event;
  15 use Civi\Api4\UserJob;
  16 use Civi\UserJob\UserJobInterface;
  17
  18 /**
  19  *
  20  * @package CRM
  21  * @copyright CiviCRM LLC https://civicrm.org/licensing
  22  */
  23 abstract class CRM_Import_Parser implements UserJobInterface {
  24   /**
  25    * Settings
  26    */
  27   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  28
  29   /**
  30    * Return codes
  31    */
  32   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  33
  34   /**
  35    * Parser modes
  36    */
  37   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  38
  39   /**
  40    * Codes for duplicate record handling
  41    */
  42   const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  43
  44   /**
  45    * Contact types
  46    */
  47   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  48
  49   /**
  50    * User job id.
  51    *
  52    * This is the primary key of the civicrm_user_job table which is used to
  53    * track the import.
  54    *
  55    * @var int
  56    */
  57   protected $userJobID;
  58
  59   /**
  60    * The user job in use.
  61    *
  62    * @var array
  63    */
  64   protected $userJob;
  65
  66   /**
  67    * Potentially ambiguous options.
  68    *
  69    * For example 'UT' is a state in more than one country.
  70    *
  71    * @var array
  72    */
  73   protected $ambiguousOptions = [];
  74
  75   /**
  76    * States to country mapping.
  77    *
  78    * @var array
  79    */
  80   protected $statesByCountry = [];
  81
  82   /**
  83    * @return int|null
  84    */
  85   public function getUserJobID(): ?int {
  86     return $this->userJobID;
  87   }
  88
  89   /**
  90    * Ids of contacts created this iteration.
  91    *
  92    * @var array
  93    */
  94   protected $createdContacts = [];
  95
  96   /**
  97    * Set user job ID.
  98    *
  99    * @param int $userJobID
 100    *
 101    * @return self
 102    */
 103   public function setUserJobID(int $userJobID): self {
 104     $this->userJobID = $userJobID;
 105     return $this;
 106   }
 107
 108   /**
 109    * Countries that the site is restricted to
 110    *
 111    * @var array|false
 112    */
 113   private $availableCountries;
 114
 115   /**
 116    *
 117    * @return array
 118    */
 119   public function getTrackingFields(): array {
 120     return [];
 121   }
 122
 123   /**
 124    * Get User Job.
 125    *
 126    * API call to retrieve the userJob row.
 127    *
 128    * @return array
 129    *
 130    * @throws \API_Exception
 131    */
 132   protected function getUserJob(): array {
 133     if (empty($this->userJob)) {
 134       $this->userJob = UserJob::get()
 135         ->addWhere('id', '=', $this->getUserJobID())
 136         ->execute()
 137         ->first();
 138     }
 139     return $this->userJob;
 140   }
 141
 142   /**
 143    * Get the relevant datasource object.
 144    *
 145    * @return \CRM_Import_DataSource|null
 146    *
 147    * @throws \API_Exception
 148    */
 149   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 150     $className = $this->getSubmittedValue('dataSource');
 151     if ($className) {
 152       /* @var CRM_Import_DataSource $dataSource */
 153       return new $className($this->getUserJobID());
 154     }
 155     return NULL;
 156   }
 157
 158   /**
 159    * Get the submitted value, as stored on the user job.
 160    *
 161    * @param string $fieldName
 162    *
 163    * @return mixed
 164    *
 165    * @noinspection PhpDocMissingThrowsInspection
 166    * @noinspection PhpUnhandledExceptionInspection
 167    */
 168   protected function getSubmittedValue(string $fieldName) {
 169     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 170   }
 171
 172   /**
 173    * Has the import completed.
 174    *
 175    * @return bool
 176    *
 177    * @throws \API_Exception
 178    * @throws \CRM_Core_Exception
 179    */
 180   public function isComplete() :bool {
 181     return $this->getDataSourceObject()->isCompleted();
 182   }
 183
 184   /**
 185    * Get configured contact type.
 186    *
 187    * @return string
 188    */
 189   protected function getContactType(): string {
 190     if (!$this->_contactType) {
 191       $contactTypeMapping = [
 192         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 193         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 194         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 195       ];
 196       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 197     }
 198     return $this->_contactType;
 199   }
 200
 201   /**
 202    * Get configured contact type.
 203    *
 204    * @return string|null
 205    */
 206   public function getContactSubType(): ?string {
 207     if (!$this->_contactSubType) {
 208       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 209     }
 210     return $this->_contactSubType;
 211   }
 212
 213   /**
 214    * Total number of non empty lines
 215    * @var int
 216    */
 217   protected $_totalCount;
 218
 219   /**
 220    * Running total number of valid lines
 221    * @var int
 222    */
 223   protected $_validCount;
 224
 225   /**
 226    * Running total number of invalid rows
 227    * @var int
 228    */
 229   protected $_invalidRowCount;
 230
 231   /**
 232    * Maximum number of non-empty/comment lines to process
 233    *
 234    * @var int
 235    */
 236   protected $_maxLinesToProcess;
 237
 238   /**
 239    * Array of error lines, bounded by MAX_ERROR
 240    * @var array
 241    */
 242   protected $_errors;
 243
 244   /**
 245    * Total number of duplicate (from database) lines
 246    * @var int
 247    */
 248   protected $_duplicateCount;
 249
 250   /**
 251    * Array of duplicate lines
 252    * @var array
 253    */
 254   protected $_duplicates;
 255
 256   /**
 257    * Maximum number of warnings to store
 258    * @var int
 259    */
 260   protected $_maxWarningCount = self::MAX_WARNINGS;
 261
 262   /**
 263    * Array of warning lines, bounded by MAX_WARNING
 264    * @var array
 265    */
 266   protected $_warnings;
 267
 268   /**
 269    * Array of all the fields that could potentially be part
 270    * of this import process
 271    * @var array
 272    */
 273   protected $_fields;
 274
 275   /**
 276    * Metadata for all available fields, keyed by unique name.
 277    *
 278    * This is intended to supercede $_fields which uses a special sauce format which
 279    * importableFieldsMetadata uses the standard getfields type format.
 280    *
 281    * @var array
 282    */
 283   protected $importableFieldsMetadata = [];
 284
 285   /**
 286    * Get metadata for all importable fields in std getfields style format.
 287    *
 288    * @return array
 289    */
 290   public function getImportableFieldsMetadata(): array {
 291     return $this->importableFieldsMetadata;
 292   }
 293
 294   /**
 295    * Set metadata for all importable fields in std getfields style format.
 296    *
 297    * @param array $importableFieldsMetadata
 298    */
 299   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 300     $this->importableFieldsMetadata = $importableFieldsMetadata;
 301   }
 302
 303   /**
 304    * Gets the fields available for importing in a key-name, title format.
 305    *
 306    * @return array
 307    *   eg. ['first_name' => 'First Name'.....]
 308    *
 309    * @throws \API_Exception
 310    *
 311    * @todo - we are constructing the metadata before we
 312    * have set the contact type so we re-do it here.
 313    *
 314    * Once we have cleaned up the way the mapper is handled
 315    * we can ditch all the existing _construct parameters in favour
 316    * of just the userJobID - there are current open PRs towards this end.
 317    */
 318   public function getAvailableFields(): array {
 319     $this->setFieldMetadata();
 320     $return = [];
 321     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 322       if ($name === 'id' && $this->isSkipDuplicates()) {
 323         // Duplicates are being skipped so id matching is not availble.
 324         continue;
 325       }
 326       $return[$name] = $field['html']['label'] ?? $field['title'];
 327     }
 328     return $return;
 329   }
 330
 331   /**
 332    * Did the user specify duplicates should be skipped and not imported.
 333    *
 334    * @return bool
 335    *
 336    * @throws \API_Exception
 337    */
 338   protected function isSkipDuplicates(): bool {
 339     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 340   }
 341
 342   /**
 343    * Is this a case where the user has opted to update existing contacts.
 344    *
 345    * @return bool
 346    */
 347   protected function isUpdateExisting(): bool {
 348     return in_array((int) $this->getSubmittedValue('onDuplicate'), [
 349       CRM_Import_Parser::DUPLICATE_UPDATE,
 350       CRM_Import_Parser::DUPLICATE_FILL,
 351     ], TRUE);
 352   }
 353
 354   /**
 355    * Did the user specify duplicates checking should be skipped, resulting in possible duplicate contacts.
 356    *
 357    * Note we still need to check for external_identifier as it will hard-fail
 358    * if we duplicate.
 359    *
 360    * @return bool
 361    */
 362   protected function isIgnoreDuplicates(): bool {
 363     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_NOCHECK;
 364   }
 365
 366   /**
 367    * Did the user specify duplicates should be filled with missing data.
 368    *
 369    * @return bool
 370    */
 371   protected function isFillDuplicates(): bool {
 372     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_FILL;
 373   }
 374
 375   /**
 376    * Array of the fields that are actually part of the import process
 377    * the position in the array also dictates their position in the import
 378    * file
 379    * @var array
 380    */
 381   protected $_activeFields = [];
 382
 383   /**
 384    * Cache the count of active fields
 385    *
 386    * @var int
 387    */
 388   protected $_activeFieldCount;
 389
 390   /**
 391    * Cache of preview rows
 392    *
 393    * @var array
 394    */
 395   protected $_rows;
 396
 397   /**
 398    * Filename of error data
 399    *
 400    * @var string
 401    */
 402   protected $_errorFileName;
 403
 404   /**
 405    * Filename of duplicate data
 406    *
 407    * @var string
 408    */
 409   protected $_duplicateFileName;
 410
 411   /**
 412    * Contact type
 413    *
 414    * @var string
 415    */
 416   public $_contactType;
 417
 418   /**
 419    * @param string $contactType
 420    *
 421    * @return CRM_Import_Parser
 422    */
 423   public function setContactType(string $contactType): CRM_Import_Parser {
 424     $this->_contactType = $contactType;
 425     return $this;
 426   }
 427
 428   /**
 429    * Contact sub-type
 430    *
 431    * @var int|null
 432    */
 433   public $_contactSubType;
 434
 435   /**
 436    * @param int|null $contactSubType
 437    *
 438    * @return self
 439    */
 440   public function setContactSubType(?int $contactSubType): self {
 441     $this->_contactSubType = $contactSubType;
 442     return $this;
 443   }
 444
 445   /**
 446    * Class constructor.
 447    */
 448   public function __construct() {
 449     $this->_maxLinesToProcess = 0;
 450   }
 451
 452   /**
 453    * Set and validate field values.
 454    *
 455    * @param array $elements
 456    *   array.
 457    */
 458   public function setActiveFieldValues($elements): void {
 459     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 460     for ($i = 0; $i < $maxCount; $i++) {
 461       $this->_activeFields[$i]->setValue($elements[$i]);
 462     }
 463
 464     // reset all the values that we did not have an equivalent import element
 465     for (; $i < $this->_activeFieldCount; $i++) {
 466       $this->_activeFields[$i]->resetValue();
 467     }
 468   }
 469
 470   /**
 471    * Format the field values for input to the api.
 472    *
 473    * @return array
 474    *   (reference) associative array of name/value pairs
 475    */
 476   public function &getActiveFieldParams() {
 477     $params = [];
 478     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 479       if (isset($this->_activeFields[$i]->_value)
 480         && !isset($params[$this->_activeFields[$i]->_name])
 481         && !isset($this->_activeFields[$i]->_related)
 482       ) {
 483
 484         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 485       }
 486     }
 487     return $params;
 488   }
 489
 490   /**
 491    * Add progress bar to the import process. Calculates time remaining, status etc.
 492    *
 493    * @param $statusID
 494    *   status id of the import process saved in $config->uploadDir.
 495    * @param bool $startImport
 496    *   True when progress bar is to be initiated.
 497    * @param $startTimestamp
 498    *   Initial timestamp when the import was started.
 499    * @param $prevTimestamp
 500    *   Previous timestamp when this function was last called.
 501    * @param $totalRowCount
 502    *   Total number of rows in the import file.
 503    *
 504    * @return NULL|$currTimestamp
 505    */
 506   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 507     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 508
 509     if ($startImport) {
 510       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 511       //do not force the browser to display the save dialog, CRM-7640
 512       $contents = json_encode([0, $status]);
 513       file_put_contents($statusFile, $contents);
 514     }
 515     else {
 516       $rowCount = $this->_rowCount ?? $this->_lineCount;
 517       $currTimestamp = time();
 518       $time = ($currTimestamp - $prevTimestamp);
 519       $recordsLeft = $totalRowCount - $rowCount;
 520       if ($recordsLeft < 0) {
 521         $recordsLeft = 0;
 522       }
 523       $estimatedTime = ($recordsLeft / 50) * $time;
 524       $estMinutes = floor($estimatedTime / 60);
 525       $timeFormatted = '';
 526       if ($estMinutes > 1) {
 527         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 528         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 529       }
 530       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 531       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 532       $statusMsg = ts('%1 of %2 records - %3 remaining',
 533         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 534       );
 535       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 536       $contents = json_encode([$processedPercent, $status]);
 537
 538       file_put_contents($statusFile, $contents);
 539       return $currTimestamp;
 540     }
 541   }
 542
 543   /**
 544    * @return array
 545    */
 546   public function getSelectValues(): array {
 547     $values = [];
 548     foreach ($this->_fields as $name => $field) {
 549       $values[$name] = $field->_title;
 550     }
 551     return $values;
 552   }
 553
 554   /**
 555    * @return array
 556    */
 557   public function getSelectTypes() {
 558     $values = [];
 559     // This is only called from the MapField form in isolation now,
 560     // so we need to set the metadata.
 561     $this->init();
 562     foreach ($this->_fields as $name => $field) {
 563       if (isset($field->_hasLocationType)) {
 564         $values[$name] = $field->_hasLocationType;
 565       }
 566     }
 567     return $values;
 568   }
 569
 570   /**
 571    * @return array
 572    */
 573   public function getHeaderPatterns(): array {
 574     $values = [];
 575     foreach ($this->importableFieldsMetadata as $name => $field) {
 576       if (isset($field['headerPattern'])) {
 577         $values[$name] = $field['headerPattern'] ?: '//';
 578       }
 579     }
 580     return $values;
 581   }
 582
 583   /**
 584    * @return array
 585    */
 586   public function getDataPatterns():array {
 587     $values = [];
 588     foreach ($this->_fields as $name => $field) {
 589       $values[$name] = $field->_dataPattern;
 590     }
 591     return $values;
 592   }
 593
 594   /**
 595    * Remove single-quote enclosures from a value array (row).
 596    *
 597    * @param array $values
 598    * @param string $enclosure
 599    *
 600    * @return void
 601    */
 602   public static function encloseScrub(&$values, $enclosure = "'") {
 603     if (empty($values)) {
 604       return;
 605     }
 606
 607     foreach ($values as $k => $v) {
 608       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 609     }
 610   }
 611
 612   /**
 613    * Setter function.
 614    *
 615    * @param int $max
 616    *
 617    * @return void
 618    */
 619   public function setMaxLinesToProcess($max) {
 620     $this->_maxLinesToProcess = $max;
 621   }
 622
 623   /**
 624    * Validate that we have the required fields to create the contact or find it to update.
 625    *
 626    * Note that the users duplicate selection affects this as follows
 627    * - if they did not select an update variant then the id field is not
 628    *   permitted in the mapping - so we can assume the presence of id means
 629    *   we should use it
 630    * - the external_identifier field is valid in place of the other fields
 631    *   when they have chosen update or fill - in this case we are only looking
 632    *   to update an existing contact.
 633    *
 634    * @param string $contactType
 635    * @param array $params
 636    * @param bool $isPermitExistingMatchFields
 637    *   True if the it is enough to have fields which will enable us to find
 638    *   an existing contact (eg. external_identifier).
 639    * @param string $prefixString
 640    *   String to include in the exception (e.g '(Child of)' if we are validating
 641    *   a related contact.
 642    *
 643    * @return void
 644    * @throws \CRM_Core_Exception
 645    */
 646   protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void {
 647     if (!empty($params['id'])) {
 648       return;
 649     }
 650     $requiredFields = [
 651       'Individual' => [
 652         'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
 653         'email' => ts('Email Address'),
 654       ],
 655       'Organization' => ['organization_name' => ts('Organization Name')],
 656       'Household' => ['household_name' => ts('Household Name')],
 657     ][$contactType];
 658     if ($isPermitExistingMatchFields) {
 659       $requiredFields['external_identifier'] = ts('External Identifier');
 660       // Historically just an email has been accepted as it is 'usually good enough'
 661       // for a dedupe rule look up - but really this is a stand in for
 662       // whatever is needed to find an existing matching contact using the
 663       // specified dedupe rule (or the default Unsupervised if not specified).
 664       $requiredFields['email'] = ts('Email Address');
 665     }
 666     $this->validateRequiredFields($requiredFields, $params, $prefixString);
 667   }
 668
 669   protected function doPostImportActions() {
 670     $userJob = $this->getUserJob();
 671     $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
 672     $actions = $userJob['metadata']['post_actions'] ?? [];
 673     if (!empty($actions['group'])) {
 674       $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']);
 675       foreach ($actions['group'] as $groupID) {
 676         $summaryInfo['groups'][$groupID]['added'] += $groupAdditions[$groupID]['added'];
 677         $summaryInfo['groups'][$groupID]['notAdded'] += $groupAdditions[$groupID]['notAdded'];
 678       }
 679     }
 680     if (!empty($actions['tag'])) {
 681       $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts, $actions['tag']);
 682       foreach ($actions['tag'] as $tagID) {
 683         $summaryInfo['tags'][$tagID]['added'] += $tagAdditions[$tagID]['added'];
 684         $summaryInfo['tags'][$tagID]['notAdded'] += $tagAdditions[$tagID]['notAdded'];
 685       }
 686     }
 687
 688     $this->userJob['metadata']['summary_info'] = $summaryInfo;
 689     UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute();
 690   }
 691
 692   public function queue() {
 693     $dataSource = $this->getDataSourceObject();
 694     $totalRowCount = $totalRows = $dataSource->getRowCount(['new']);
 695     $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']);
 696     $offset = 0;
 697     $batchSize = 5;
 698     while ($totalRows > 0) {
 699       if ($totalRows < $batchSize) {
 700         $batchSize = $totalRows;
 701       }
 702       $task = new CRM_Queue_Task(
 703         [get_class($this), 'runJob'],
 704         // Offset is unused by our import classes, but required by the interface.
 705         ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize, 'offset' => 0],
 706         ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount])
 707       );
 708       $queue->createItem($task);
 709       $totalRows -= $batchSize;
 710       $offset += $batchSize;
 711     }
 712
 713   }
 714
 715   /**
 716    * Add imported contacts to groups.
 717    *
 718    * @param array $contactIDs
 719    * @param array $groups
 720    *
 721    * @return array
 722    */
 723   private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array {
 724     $groupAdditions = [];
 725     foreach ($groups as $groupID) {
 726       // @todo - this function has been in use historically but it does not seem
 727       // to add much efficiency of get + create api calls
 728       // and it doesn't give enough control over cache flushing for smaller batches.
 729       // Note that the import updates a lot of enities & checking & updating the group
 730       // shouldn't add much performance wise. However, cache flushing will
 731       $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIDs, $groupID);
 732       $groupAdditions[$groupID] = [
 733         'added' => (int) $addCount[1],
 734         'notAdded' => (int) $addCount[2],
 735       ];
 736     }
 737     return $groupAdditions;
 738   }
 739
 740   /**
 741    * Tag imported contacts.
 742    *
 743    * @param array $contactIDs
 744    * @param array $tags
 745    *
 746    * @return array
 747    */
 748   private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) {
 749     $tagAdditions = [];
 750     foreach ($tags as $tagID) {
 751       // @todo - this function has been in use historically but it does not seem
 752       // to add much efficiency of get + create api calls
 753       // and it doesn't give enough control over cache flushing for smaller batches.
 754       // Note that the import updates a lot of enities & checking & updating the group
 755       // shouldn't add much performance wise. However, cache flushing will
 756       $outcome = CRM_Core_BAO_EntityTag::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE);
 757       $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]];
 758     }
 759     return $tagAdditions;
 760   }
 761
 762   /**
 763    * Determines the file extension based on error code.
 764    *
 765    * @var int $type error code constant
 766    * @return string
 767    */
 768   public static function errorFileName($type) {
 769     $fileName = NULL;
 770     if (empty($type)) {
 771       return $fileName;
 772     }
 773
 774     $config = CRM_Core_Config::singleton();
 775     $fileName = $config->uploadDir . "sqlImport";
 776     switch ($type) {
 777       case self::ERROR:
 778         $fileName .= '.errors';
 779         break;
 780
 781       case self::DUPLICATE:
 782         $fileName .= '.duplicates';
 783         break;
 784
 785       case self::NO_MATCH:
 786         $fileName .= '.mismatch';
 787         break;
 788
 789       case self::UNPARSED_ADDRESS_WARNING:
 790         $fileName .= '.unparsedAddress';
 791         break;
 792     }
 793
 794     return $fileName;
 795   }
 796
 797   /**
 798    * Determines the file name based on error code.
 799    *
 800    * @var int $type code constant
 801    * @return string
 802    */
 803   public static function saveFileName($type) {
 804     $fileName = NULL;
 805     if (empty($type)) {
 806       return $fileName;
 807     }
 808     switch ($type) {
 809       case self::ERROR:
 810         $fileName = 'Import_Errors.csv';
 811         break;
 812
 813       case self::DUPLICATE:
 814         $fileName = 'Import_Duplicates.csv';
 815         break;
 816
 817       case self::NO_MATCH:
 818         $fileName = 'Import_Mismatch.csv';
 819         break;
 820
 821       case self::UNPARSED_ADDRESS_WARNING:
 822         $fileName = 'Import_Unparsed_Address.csv';
 823         break;
 824     }
 825
 826     return $fileName;
 827   }
 828
 829   /**
 830    * Check if contact is a duplicate .
 831    *
 832    * @param array $formatValues
 833    *
 834    * @return array
 835    */
 836   protected function checkContactDuplicate(&$formatValues) {
 837     //retrieve contact id using contact dedupe rule
 838     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->getContactType();
 839     $formatValues['version'] = 3;
 840     require_once 'CRM/Utils/DeprecatedUtils.php';
 841     $params = $formatValues;
 842     static $cIndieFields = NULL;
 843     static $defaultLocationId = NULL;
 844
 845     $contactType = $params['contact_type'];
 846     if ($cIndieFields == NULL) {
 847       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 848       $cIndieFields = $cTempIndieFields;
 849
 850       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 851
 852       // set the value to default location id else set to 1
 853       if (!$defaultLocationId = (int) $defaultLocation->id) {
 854         $defaultLocationId = 1;
 855       }
 856     }
 857
 858     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 859
 860     $contactFormatted = [];
 861     foreach ($params as $key => $field) {
 862       if ($field == NULL || $field === '') {
 863         continue;
 864       }
 865       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 866       // instead of soft credit contact.
 867       if (is_array($field) && $key !== "soft_credit") {
 868         foreach ($field as $value) {
 869           $break = FALSE;
 870           if (is_array($value)) {
 871             foreach ($value as $name => $testForEmpty) {
 872               if ($name !== 'phone_type' &&
 873                 ($testForEmpty === '' || $testForEmpty == NULL)
 874               ) {
 875                 $break = TRUE;
 876                 break;
 877               }
 878             }
 879           }
 880           else {
 881             $break = TRUE;
 882           }
 883           if (!$break) {
 884             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 885           }
 886         }
 887         continue;
 888       }
 889
 890       $value = [$key => $field];
 891
 892       // check if location related field, then we need to add primary location type
 893       if (in_array($key, $locationFields)) {
 894         $value['location_type_id'] = $defaultLocationId;
 895       }
 896       elseif (array_key_exists($key, $cIndieFields)) {
 897         $value['contact_type'] = $contactType;
 898       }
 899
 900       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 901     }
 902
 903     $contactFormatted['contact_type'] = $contactType;
 904
 905     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 906   }
 907
 908   /**
 909    * This function adds the contact variable in $values to the
 910    * parameter list $params.  For most cases, $values should have length 1.  If
 911    * the variable being added is a child of Location, a location_type_id must
 912    * also be included.  If it is a child of phone, a phone_type must be included.
 913    *
 914    * @param array $values
 915    *   The variable(s) to be added.
 916    * @param array $params
 917    *   The structured parameter list.
 918    *
 919    * @return bool|CRM_Utils_Error
 920    */
 921   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 922     // @todo - like most functions in import ... most of this is cruft....
 923     // Crawl through the possible classes:
 924     // Contact
 925     //      Individual
 926     //      Household
 927     //      Organization
 928     //          Location
 929     //              Address
 930     //              Email
 931     //              Phone
 932     //              IM
 933     //      Note
 934     //      Custom
 935
 936     // Cache the various object fields
 937     static $fields = NULL;
 938
 939     if ($fields == NULL) {
 940       $fields = [];
 941     }
 942
 943     // first add core contact values since for other Civi modules they are not added
 944     require_once 'CRM/Contact/BAO/Contact.php';
 945     $contactFields = CRM_Contact_DAO_Contact::fields();
 946     _civicrm_api3_store_values($contactFields, $values, $params);
 947
 948     if (isset($values['contact_type'])) {
 949       // we're an individual/household/org property
 950
 951       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 952
 953       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 954       return TRUE;
 955     }
 956
 957     if (isset($values['individual_prefix'])) {
 958       if (!empty($params['prefix_id'])) {
 959         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 960         $params['prefix'] = $prefixes[$params['prefix_id']];
 961       }
 962       else {
 963         $params['prefix'] = $values['individual_prefix'];
 964       }
 965       return TRUE;
 966     }
 967
 968     if (isset($values['individual_suffix'])) {
 969       if (!empty($params['suffix_id'])) {
 970         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 971         $params['suffix'] = $suffixes[$params['suffix_id']];
 972       }
 973       else {
 974         $params['suffix'] = $values['individual_suffix'];
 975       }
 976       return TRUE;
 977     }
 978
 979     if (isset($values['gender'])) {
 980       if (!empty($params['gender_id'])) {
 981         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 982         $params['gender'] = $genders[$params['gender_id']];
 983       }
 984       else {
 985         $params['gender'] = $values['gender'];
 986       }
 987       return TRUE;
 988     }
 989
 990     // format the website params.
 991     if (!empty($values['url'])) {
 992       static $websiteFields;
 993       if (!is_array($websiteFields)) {
 994         require_once 'CRM/Core/DAO/Website.php';
 995         $websiteFields = CRM_Core_DAO_Website::fields();
 996       }
 997       if (!array_key_exists('website', $params) ||
 998         !is_array($params['website'])
 999       ) {
1000         $params['website'] = [];
1001       }
1002
1003       $websiteCount = count($params['website']);
1004       _civicrm_api3_store_values($websiteFields, $values,
1005         $params['website'][++$websiteCount]
1006       );
1007
1008       return TRUE;
1009     }
1010
1011     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1012     if (!empty($values['location_type_id'])) {
1013       static $fields = NULL;
1014       if ($fields == NULL) {
1015         $fields = [];
1016       }
1017
1018       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
1019         $name = strtolower($block);
1020         if (!array_key_exists($name, $values)) {
1021           continue;
1022         }
1023
1024         if ($name === 'phone_ext') {
1025           $block = 'Phone';
1026         }
1027
1028         // block present in value array.
1029         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
1030           $params[$name] = [];
1031         }
1032
1033         if (!array_key_exists($block, $fields)) {
1034           $className = "CRM_Core_DAO_$block";
1035           $fields[$block] =& $className::fields();
1036         }
1037
1038         $blockCnt = count($params[$name]);
1039
1040         // copy value to dao field name.
1041         if ($name == 'im') {
1042           $values['name'] = $values[$name];
1043         }
1044
1045         _civicrm_api3_store_values($fields[$block], $values,
1046           $params[$name][++$blockCnt]
1047         );
1048
1049         if (empty($params['id']) && ($blockCnt == 1)) {
1050           $params[$name][$blockCnt]['is_primary'] = TRUE;
1051         }
1052
1053         // we only process single block at a time.
1054         return TRUE;
1055       }
1056
1057       // handle address fields.
1058       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1059         $params['address'] = [];
1060       }
1061
1062       $addressCnt = 1;
1063       foreach ($params['address'] as $cnt => $addressBlock) {
1064         if (CRM_Utils_Array::value('location_type_id', $values) ==
1065           CRM_Utils_Array::value('location_type_id', $addressBlock)
1066         ) {
1067           $addressCnt = $cnt;
1068           break;
1069         }
1070         $addressCnt++;
1071       }
1072
1073       if (!array_key_exists('Address', $fields)) {
1074         $fields['Address'] = CRM_Core_DAO_Address::fields();
1075       }
1076
1077       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1078       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1079       // the address in CRM_Core_BAO_Address::create method
1080       if (!empty($values['location_type_id'])) {
1081         static $customFields = [];
1082         if (empty($customFields)) {
1083           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1084         }
1085         // make a copy of values, as we going to make changes
1086         $newValues = $values;
1087         foreach ($values as $key => $val) {
1088           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1089           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1090             // mark an entry in fields array since we want the value of custom field to be copied
1091             $fields['Address'][$key] = NULL;
1092
1093             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1094             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1095               $mulValues = explode(',', $val);
1096               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1097               $newValues[$key] = [];
1098               foreach ($mulValues as $v1) {
1099                 foreach ($customOption as $v2) {
1100                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1101                     (strtolower($v2['value']) == strtolower(trim($v1)))
1102                   ) {
1103                     if ($htmlType == 'CheckBox') {
1104                       $newValues[$key][$v2['value']] = 1;
1105                     }
1106                     else {
1107                       $newValues[$key][] = $v2['value'];
1108                     }
1109                   }
1110                 }
1111               }
1112             }
1113           }
1114         }
1115         // consider new values
1116         $values = $newValues;
1117       }
1118
1119       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1120
1121       $addressFields = [
1122         'county',
1123         'country',
1124         'state_province',
1125         'supplemental_address_1',
1126         'supplemental_address_2',
1127         'supplemental_address_3',
1128         'StateProvince.name',
1129       ];
1130
1131       foreach ($addressFields as $field) {
1132         if (array_key_exists($field, $values)) {
1133           if (!array_key_exists('address', $params)) {
1134             $params['address'] = [];
1135           }
1136           $params['address'][$addressCnt][$field] = $values[$field];
1137         }
1138       }
1139
1140       if ($addressCnt == 1) {
1141
1142         $params['address'][$addressCnt]['is_primary'] = TRUE;
1143       }
1144       return TRUE;
1145     }
1146
1147     if (isset($values['note'])) {
1148       // add a note field
1149       if (!isset($params['note'])) {
1150         $params['note'] = [];
1151       }
1152       $noteBlock = count($params['note']) + 1;
1153
1154       $params['note'][$noteBlock] = [];
1155       if (!isset($fields['Note'])) {
1156         $fields['Note'] = CRM_Core_DAO_Note::fields();
1157       }
1158
1159       // get the current logged in civicrm user
1160       $session = CRM_Core_Session::singleton();
1161       $userID = $session->get('userID');
1162
1163       if ($userID) {
1164         $values['contact_id'] = $userID;
1165       }
1166
1167       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1168
1169       return TRUE;
1170     }
1171
1172     // Check for custom field values
1173
1174     if (empty($fields['custom'])) {
1175       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1176         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1177       );
1178     }
1179
1180     foreach ($values as $key => $value) {
1181       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1182         // check if it's a valid custom field id
1183
1184         if (!array_key_exists($customFieldID, $fields['custom'])) {
1185           return civicrm_api3_create_error('Invalid custom field ID');
1186         }
1187         else {
1188           $params[$key] = $value;
1189         }
1190       }
1191     }
1192   }
1193
1194   /**
1195    * Parse a field which could be represented by a label or name value rather than the DB value.
1196    *
1197    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1198    *
1199    * but if not available then see if we have a label that can be converted to a name.
1200    *
1201    * @param string|int|null $submittedValue
1202    * @param array $fieldSpec
1203    *   Metadata for the field
1204    *
1205    * @return mixed
1206    */
1207   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1208     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1209     if (!isset($fieldSpec['bao'])) {
1210       return $submittedValue;
1211     }
1212     /* @var \CRM_Core_DAO $bao */
1213     $bao = $fieldSpec['bao'];
1214     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1215     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1216     if (isset($nameOptions[$submittedValue])) {
1217       return $submittedValue;
1218     }
1219     if (in_array($submittedValue, $nameOptions)) {
1220       return array_search($submittedValue, $nameOptions, TRUE);
1221     }
1222
1223     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1224     if (isset($labelOptions[$submittedValue])) {
1225       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1226     }
1227     return '';
1228   }
1229
1230   /**
1231    * This is code extracted from 4 places where this exact snippet was being duplicated.
1232    *
1233    * FIXME: Extracting this was a first step, but there's also
1234    *  1. Inconsistency in the way other select options are handled.
1235    *     Contribution adds handling for Select/Radio/Autocomplete
1236    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1237    *     Membership is missing all of it
1238    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1239    *
1240    * @param $customFieldID
1241    * @param $value
1242    * @param $fieldType
1243    * @return array
1244    */
1245   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1246     $mulValues = explode(',', $value);
1247     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1248     $values = [];
1249     foreach ($mulValues as $v1) {
1250       foreach ($customOption as $customValueID => $customLabel) {
1251         $customValue = $customLabel['value'];
1252         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1253           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1254         ) {
1255           $values[] = $customValue;
1256         }
1257       }
1258     }
1259     return $values;
1260   }
1261
1262   /**
1263    * Validate that the field requirements are met in the params.
1264    *
1265    * @param array $requiredFields
1266    * @param array $params
1267    *   An array of required fields (fieldName => label)
1268    *   - note this follows the and / or array nesting we see in permission checks
1269    *   eg.
1270    *   [
1271    *     'email' => ts('Email'),
1272    *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1273    *   ]
1274    *   Means 'email' OR 'first_name AND 'last_name'.
1275    * @param string $prefixString
1276    *
1277    * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1278    */
1279   protected function validateRequiredFields(array $requiredFields, array $params, $prefixString = ''): void {
1280     if (empty($requiredFields)) {
1281       return;
1282     }
1283     $missingFields = [];
1284     foreach ($requiredFields as $key => $required) {
1285       if (!is_array($required)) {
1286         $importParameter = $params[$key] ?? [];
1287         if (!is_array($importParameter)) {
1288           if (!empty($importParameter)) {
1289             return;
1290           }
1291         }
1292         else {
1293           foreach ($importParameter as $locationValues) {
1294             if (!empty($locationValues[$key])) {
1295               return;
1296             }
1297           }
1298         }
1299
1300         $missingFields[$key] = $required;
1301       }
1302       else {
1303         foreach ($required as $field => $label) {
1304           if (empty($params[$field])) {
1305             $missing[$field] = $label;
1306           }
1307         }
1308         if (empty($missing)) {
1309           return;
1310         }
1311         $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1312       }
1313     }
1314     throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1315   }
1316
1317   /**
1318    * Get the field value, transformed by metadata.
1319    *
1320    * @param string $fieldName
1321    * @param string|int $importedValue
1322    *   Value as it came in from the datasource.
1323    *
1324    * @return string|array|bool|int
1325    * @throws \API_Exception
1326    */
1327   protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1328     if (empty($importedValue)) {
1329       return $importedValue;
1330     }
1331     $fieldMetadata = $this->getFieldMetadata($fieldName);
1332     if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) {
1333       $values = [];
1334       foreach (explode(',', $importedValue) as $value) {
1335         $values[] = $this->getTransformedFieldValue($fieldName, trim($value));
1336       }
1337       return $values;
1338     }
1339     if ($fieldName === 'url') {
1340       return CRM_Utils_Rule::url($importedValue) ? $importedValue : 'invalid_import_value';
1341     }
1342
1343     if ($fieldName === 'email') {
1344       return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value';
1345     }
1346
1347     if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) {
1348       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1349     }
1350     if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) {
1351       return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value';
1352     }
1353     if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) {
1354       $value = CRM_Utils_String::strtoboolstr($importedValue);
1355       if ($value !== FALSE) {
1356         return (bool) $value;
1357       }
1358       return 'invalid_import_value';
1359     }
1360     if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) {
1361       $value = CRM_Utils_Date::formatDate($importedValue, (int) $this->getSubmittedValue('dateFormats'));
1362       return $value ?: 'invalid_import_value';
1363     }
1364     $options = $this->getFieldOptions($fieldName);
1365     if ($options !== FALSE) {
1366       if ($this->isAmbiguous($fieldName, $importedValue)) {
1367         // We can't transform it at this stage. Perhaps later we can with
1368         // other information such as country.
1369         return $importedValue;
1370       }
1371
1372       $comparisonValue = $this->getComparisonValue($importedValue);
1373       return $options[$comparisonValue] ?? 'invalid_import_value';
1374     }
1375     if (!empty($fieldMetadata['FKClassName']) || !empty($fieldMetadata['pseudoconstant']['prefetch'])) {
1376       // @todo - make this generic - for fields where getOptions doesn't fetch
1377       // getOptions does not retrieve these fields with high potential results
1378       if ($fieldName === 'event_id') {
1379         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1380           $event = Event::get()->addClause('OR', ['title', '=', $importedValue], ['id', '=', $importedValue])->addSelect('id')->execute()->first();
1381           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $event['id'] ?? FALSE;
1382         }
1383         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1384       }
1385       if ($fieldMetadata['name'] === 'campaign_id') {
1386         if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) {
1387           $campaign = Campaign::get()->addClause('OR', ['title', '=', $importedValue], ['name', '=', $importedValue])->addSelect('id')->execute()->first();
1388           Civi::$statics[__CLASS__][$fieldName][$importedValue] = $campaign['id'] ?? FALSE;
1389         }
1390         return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value';
1391       }
1392     }
1393     if ($fieldMetadata['type'] === CRM_Utils_Type::T_INT) {
1394       // We have resolved the options now so any remaining ones should be integers.
1395       return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value';
1396     }
1397     return $importedValue;
1398   }
1399
1400   /**
1401    * @param string $fieldName
1402    *
1403    * @return false|array
1404    *
1405    * @throws \API_Exception
1406    */
1407   protected function getFieldOptions(string $fieldName) {
1408     return $this->getFieldMetadata($fieldName, TRUE)['options'];
1409   }
1410
1411   /**
1412    * Get the metadata for the field.
1413    *
1414    * @param string $fieldName
1415    * @param bool $loadOptions
1416    * @param bool $limitToContactType
1417    *   Only show fields for the type to import (not appropriate when looking up
1418    *   related contact fields).
1419    *
1420    * @return array
1421    *
1422    * @noinspection PhpDocMissingThrowsInspection
1423    * @noinspection PhpUnhandledExceptionInspection
1424    */
1425   protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1426
1427     $fieldMap = $this->getOddlyMappedMetadataFields();
1428     $fieldMapName = empty($fieldMap[$fieldName]) ? $fieldName : $fieldMap[$fieldName];
1429
1430     // This whole business of only loading metadata for one type when we actually need it for all is ... dubious.
1431     if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) {
1432       if ($loadOptions || !$limitToContactType) {
1433         $this->importableFieldsMetadata[$fieldMapName] = CRM_Contact_BAO_Contact::importableFields('All')[$fieldMapName];
1434       }
1435     }
1436
1437     $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName];
1438     if ($loadOptions && !isset($fieldMetadata['options'])) {
1439       if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') {
1440         // Probably already loaded and also supports abbreviations - eg. NSW.
1441         // Supporting for core AND custom state fields is more consistent.
1442         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id');
1443         return $this->importableFieldsMetadata[$fieldMapName];
1444       }
1445       if (($fieldMetadata['data_type'] ?? '') === 'Country') {
1446         // Probably already loaded and also supports abbreviations - eg. NSW.
1447         // Supporting for core AND custom state fields is more consistent.
1448         $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('country_id');
1449         return $this->importableFieldsMetadata[$fieldMapName];
1450       }
1451       $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;
1452
1453       if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
1454         $options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
1455           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1456           'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
1457           'select' => ['options'],
1458         ])->first()['options'];
1459       }
1460       else {
1461         if (!empty($fieldMetadata['custom_group_id'])) {
1462           $customField = CustomField::get(FALSE)
1463             ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
1464             ->addSelect('name', 'custom_group_id.name')
1465             ->execute()
1466             ->first();
1467           $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
1468         }
1469         $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1470           'loadOptions' => ['id', 'name', 'label', 'abbr'],
1471           'where' => [['name', '=', $optionFieldName]],
1472           'select' => ['options'],
1473         ])->first()['options'];
1474       }
1475       if (is_array($options)) {
1476         // We create an array of the possible variants - notably including
1477         // name AND label as either might be used. We also lower case before checking
1478         $values = [];
1479         foreach ($options as $option) {
1480           $idKey = $this->getComparisonValue($option['id']);
1481           $values[$idKey] = $option['id'];
1482           foreach (['name', 'label', 'abbr'] as $key) {
1483             $optionValue = $this->getComparisonValue($option[$key] ?? '');
1484             if ($optionValue !== '') {
1485               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
1486                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
1487                   $this->ambiguousOptions[$fieldName][$optionValue] = [$values[$optionValue]];
1488                 }
1489                 $this->ambiguousOptions[$fieldName][$optionValue][] = $option['id'];
1490               }
1491               else {
1492                 $values[$optionValue] = $option['id'];
1493               }
1494             }
1495           }
1496         }
1497         $this->importableFieldsMetadata[$fieldMapName]['options'] = $values;
1498       }
1499       else {
1500         $this->importableFieldsMetadata[$fieldMapName]['options'] = $options ?: FALSE;
1501       }
1502       return $this->importableFieldsMetadata[$fieldMapName];
1503     }
1504     return $fieldMetadata;
1505   }
1506
1507   /**
1508    * Get the field metadata for fields to be be offered to match the contact.
1509    *
1510    * @return array
1511    * @noinspection PhpDocMissingThrowsInspection
1512    */
1513   protected function getContactMatchingFields(): array {
1514     $contactFields = CRM_Contact_BAO_Contact::importableFields($this->getContactType(), NULL);
1515     $fields = ['external_identifier' => $contactFields['external_identifier']];
1516     $fields['external_identifier']['title'] .= ' (match to contact)';
1517     // Using new Dedupe rule.
1518     $ruleParams = [
1519       'contact_type' => $this->getContactType(),
1520       'used' => $this->getSubmittedValue('dedupe_rule_id') ?? 'Unsupervised',
1521     ];
1522     $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
1523
1524     if (is_array($fieldsArray)) {
1525       foreach ($fieldsArray as $value) {
1526         $customFieldId = CRM_Core_DAO::getFieldValue('CRM_Core_DAO_CustomField',
1527           $value,
1528           'id',
1529           'column_name'
1530         );
1531         $value = trim($customFieldId ? 'custom_' . $customFieldId : $value);
1532         $fields[$value] = $contactFields[$value] ?? NULL;
1533         $title = $fields[$value]['title'] . ' (match to contact)';
1534         $fields[$value]['title'] = $title;
1535       }
1536     }
1537     return $fields;
1538   }
1539
1540   /**
1541    * @param $customFieldID
1542    * @param $value
1543    * @param array $fieldMetaData
1544    * @param $dateType
1545    *
1546    * @return ?string
1547    */
1548   protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?string {
1549     /* validate the data against the CF type */
1550
1551     if ($value) {
1552       $dataType = $fieldMetaData['data_type'];
1553       $htmlType = $fieldMetaData['html_type'];
1554       $isSerialized = CRM_Core_BAO_CustomField::isSerialized($fieldMetaData);
1555       if ($dataType === 'Date') {
1556         $params = ['date_field' => $value];
1557         if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, 'date_field')) {
1558           return NULL;
1559         }
1560         return $fieldMetaData['label'];
1561       }
1562       elseif ($dataType === 'Boolean') {
1563         if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1564           return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1565         }
1566       }
1567       // need not check for label filed import
1568       $selectHtmlTypes = [
1569         'CheckBox',
1570         'Select',
1571         'Radio',
1572       ];
1573       if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1574         $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1575         if (!$valid) {
1576           return $fieldMetaData['label'];
1577         }
1578       }
1579
1580       // check for values for custom fields for checkboxes and multiselect
1581       if ($isSerialized && $dataType != 'ContactReference') {
1582         $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1583         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1584         foreach ($mulValues as $v1) {
1585
1586           $flag = FALSE;
1587           foreach ($customOption as $v2) {
1588             if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1589               $flag = TRUE;
1590             }
1591           }
1592
1593           if (!$flag) {
1594             return $fieldMetaData['label'];
1595           }
1596         }
1597       }
1598       elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1599         $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1600         $flag = FALSE;
1601         foreach ($customOption as $v2) {
1602           if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1603             $flag = TRUE;
1604           }
1605         }
1606         if (!$flag) {
1607           return $fieldMetaData['label'];
1608         }
1609       }
1610     }
1611
1612     return NULL;
1613   }
1614
1615   /**
1616    * Get the entity for the given field.
1617    *
1618    * @param string $fieldName
1619    *
1620    * @return mixed|null
1621    * @throws \API_Exception
1622    */
1623   protected function getFieldEntity(string $fieldName) {
1624     if ($fieldName === 'do_not_import') {
1625       return NULL;
1626     }
1627     if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) {
1628       return 'Contact';
1629     }
1630     $metadata = $this->getFieldMetadata($fieldName);
1631     if (!isset($metadata['entity'])) {
1632       return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ? 'Contact' : $metadata['extends'];
1633     }
1634
1635     // Our metadata for these is fugly. Handling the fugliness during retrieval.
1636     if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1637       return 'Address';
1638     }
1639     return $metadata['entity'];
1640   }
1641
1642   /**
1643    * Validate the import file, updating the import table with results.
1644    *
1645    * @throws \API_Exception
1646    * @throws \CRM_Core_Exception
1647    */
1648   public function validate(): void {
1649     $dataSource = $this->getDataSourceObject();
1650     while ($row = $dataSource->getRow()) {
1651       try {
1652         $rowNumber = $row['_id'];
1653         $values = array_values($row);
1654         $this->validateValues($values);
1655         $this->setImportStatus($rowNumber, 'NEW', '');
1656       }
1657       catch (CRM_Core_Exception $e) {
1658         $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
1659       }
1660     }
1661   }
1662
1663   /**
1664    * Validate the import values.
1665    *
1666    * The values array represents a row in the datasource.
1667    *
1668    * @param array $values
1669    *
1670    * @throws \API_Exception
1671    * @throws \CRM_Core_Exception
1672    */
1673   public function validateValues(array $values): void {
1674     $params = $this->getMappedRow($values);
1675     $this->validateParams($params);
1676   }
1677
1678   /**
1679    * @param array $params
1680    *
1681    * @throws \CRM_Core_Exception
1682    */
1683   protected function validateParams(array $params): void {
1684     if (empty($params['id'])) {
1685       $this->validateRequiredFields($this->getRequiredFields(), $params);
1686     }
1687     $errors = [];
1688     foreach ($params as $key => $value) {
1689       $errors = array_merge($this->getInvalidValues($value, $key), $errors);
1690     }
1691     if ($errors) {
1692       throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors));
1693     }
1694   }
1695
1696   /**
1697    * Search the value for the string 'invalid_import_value'.
1698    *
1699    * If the string is found it indicates the fields was rejected
1700    * during `getTransformedValue` as not having valid data.
1701    *
1702    * @param string|array|int $value
1703    * @param string $key
1704    * @param string $prefixString
1705    *
1706    * @return array
1707    */
1708   protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
1709     $errors = [];
1710     if ($value === 'invalid_import_value') {
1711       if (!is_numeric($key)) {
1712         $metadata = $this->getFieldMetadata($key);
1713         $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1714       }
1715       else {
1716         // Numeric key suggests we are drilling into option values
1717         $errors[] = TRUE;
1718       }
1719     }
1720     elseif (is_array($value)) {
1721       foreach ($value as $innerKey => $innerValue) {
1722         $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1723         if ($result === [TRUE]) {
1724           $metadata = $this->getFieldMetadata($key);
1725           $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']);
1726         }
1727         elseif (!empty($result)) {
1728           $errors = array_merge($result, $errors);
1729         }
1730       }
1731     }
1732     return array_filter($errors);
1733   }
1734
1735   /**
1736    * Get the available countries.
1737    *
1738    * If the site is not configured with a restriction then all countries are valid
1739    * but otherwise only a select array are.
1740    *
1741    * @return array|false
1742    *   FALSE indicates no restrictions.
1743    */
1744   protected function getAvailableCountries() {
1745     if ($this->availableCountries === NULL) {
1746       $availableCountries = Civi::settings()->get('countryLimit');
1747       $this->availableCountries = !empty($availableCountries) ? array_fill_keys($availableCountries, TRUE) : FALSE;
1748     }
1749     return $this->availableCountries;
1750   }
1751
1752   /**
1753    * Get the metadata field for which importable fields does not key the actual field name.
1754    *
1755    * @return string[]
1756    */
1757   protected function getOddlyMappedMetadataFields(): array {
1758     return [
1759       'country_id' => 'country',
1760       'state_province_id' => 'state_province',
1761       'county_id' => 'county',
1762       'email_greeting_id' => 'email_greeting',
1763       'postal_greeting_id' => 'postal_greeting',
1764       'addressee_id' => 'addressee',
1765     ];
1766   }
1767
1768   /**
1769    * Get the default country for the site.
1770    *
1771    * @return int
1772    */
1773   protected function getSiteDefaultCountry(): int {
1774     if (!isset($this->siteDefaultCountry)) {
1775       $this->siteDefaultCountry = (int) Civi::settings()->get('defaultContactCountry');
1776     }
1777     return $this->siteDefaultCountry;
1778   }
1779
1780   /**
1781    * Is the option ambiguous.
1782    *
1783    * @param string $fieldName
1784    * @param string $importedValue
1785    */
1786   protected function isAmbiguous(string $fieldName, $importedValue): bool {
1787     return !empty($this->ambiguousOptions[$fieldName][$this->getComparisonValue($importedValue)]);
1788   }
1789
1790   /**
1791    * Get the civicrm_mapping_field appropriate layout for the mapper input.
1792    *
1793    * For simple parsers (not contribution or contact) the input looks like
1794    * ['first_name', 'custom_32']
1795    * and it is converted to
1796    *
1797    *  ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5],
1798    *
1799    * @param array $fieldMapping
1800    * @param int $mappingID
1801    * @param int $columnNumber
1802    *
1803    * @return array
1804    */
1805   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
1806     return [
1807       'name' => $fieldMapping[0],
1808       'mapping_id' => $mappingID,
1809       'column_number' => $columnNumber,
1810     ];
1811   }
1812
1813   /**
1814    * The initializer code, called before the processing
1815    *
1816    * @return void
1817    */
1818   public function init() {
1819     // Force re-load of user job.
1820     unset($this->userJob);
1821     $this->setFieldMetadata();
1822   }
1823
1824   /**
1825    * @param array $mappedField
1826    *   Field detail as would be saved in field_mapping table
1827    *   or as returned from getMappingFieldFromMapperInput
1828    *
1829    * @return string
1830    * @throws \API_Exception
1831    */
1832   public function getMappedFieldLabel(array $mappedField): string {
1833     // doNotImport is on it's way out - skip fields will be '' once all is done.
1834     if ($mappedField['name'] === 'doNotImport') {
1835       return '';
1836     }
1837     $this->setFieldMetadata();
1838     $metadata = $this->getFieldMetadata($mappedField['name']);
1839     return $metadata['html']['label'] ?? $metadata['title'];
1840   }
1841
1842   /**
1843    * Get the row from the csv mapped to our parameters.
1844    *
1845    * @param array $values
1846    *
1847    * @return array
1848    * @throws \API_Exception
1849    */
1850   public function getMappedRow(array $values): array {
1851     $params = [];
1852     foreach ($this->getFieldMappings() as $i => $mappedField) {
1853       if ($mappedField['name'] === 'do_not_import') {
1854         continue;
1855       }
1856       if ($mappedField['name']) {
1857         $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
1858       }
1859     }
1860     return $params;
1861   }
1862
1863   /**
1864    * Get the field mappings for the import.
1865    *
1866    * This is the same format as saved in civicrm_mapping_field except
1867    * that location_type_id = 'Primary' rather than empty where relevant.
1868    * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
1869    *
1870    * @return array
1871    * @throws \API_Exception
1872    */
1873   protected function getFieldMappings(): array {
1874     $mappedFields = [];
1875     $mapper = $this->getSubmittedValue('mapper');
1876     foreach ($mapper as $i => $mapperRow) {
1877       $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
1878       // Just for clarity since 0 is a pseudo-value
1879       unset($mappedField['mapping_id']);
1880       $mappedFields[] = $mappedField;
1881     }
1882     return $mappedFields;
1883   }
1884
1885   /**
1886    * Run import.
1887    *
1888    * @param \CRM_Queue_TaskContext $taskContext
1889    *
1890    * @param int $userJobID
1891    * @param int $limit
1892    * @param int $offset
1893    *
1894    * @return bool
1895    * @throws \API_Exception
1896    * @throws \CRM_Core_Exception
1897    */
1898   public static function runJob(\CRM_Queue_TaskContext $taskContext, int $userJobID, int $limit, int $offset): bool {
1899     $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('job_type')->execute()->first();
1900     $parserClass = NULL;
1901     foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
1902       if ($userJob['job_type'] === $userJobType['id']) {
1903         $parserClass = $userJobType['class'];
1904       }
1905     }
1906     /* @var \CRM_Import_Parser $parser */
1907     $parser = new $parserClass();
1908     $parser->setUserJobID($userJobID);
1909     // Not sure if we still need to init....
1910     $parser->init();
1911     $dataSource = $parser->getDataSourceObject();
1912     $dataSource->setStatuses(['new']);
1913     $dataSource->setLimit($limit);
1914
1915     while ($row = $dataSource->getRow()) {
1916       $values = array_values($row);
1917       $parser->import($values);
1918     }
1919     $parser->doPostImportActions();
1920     return TRUE;
1921   }
1922
1923   /**
1924    * Check if an error in custom data.
1925    *
1926    * @deprecated all of this is duplicated if getTransformedValue is used.
1927    *
1928    * @param array $params
1929    * @param string $errorMessage
1930    *   A string containing all the error-fields.
1931    *
1932    * @param null $csType
1933    */
1934   public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) {
1935     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1936     $errors = [];
1937
1938     if (!empty($params['contact_sub_type'])) {
1939       $csType = $params['contact_sub_type'] ?? NULL;
1940     }
1941
1942     if (empty($params['contact_type'])) {
1943       $params['contact_type'] = 'Individual';
1944     }
1945
1946     // get array of subtypes - CRM-18708
1947     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1948       $csType = $this->getSubtypes($params['contact_type']);
1949     }
1950
1951     if (is_array($csType)) {
1952       // fetch custom fields for every subtype and add it to $customFields array
1953       // CRM-18708
1954       $customFields = [];
1955       foreach ($csType as $cType) {
1956         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1957       }
1958     }
1959     else {
1960       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1961     }
1962
1963     foreach ($params as $key => $value) {
1964       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1965         //For address custom fields, we do get actual custom field value as an inner array of
1966         //values so need to modify
1967         if (!array_key_exists($customFieldID, $customFields)) {
1968           return ts('field ID');
1969         }
1970         /* check if it's a valid custom field id */
1971         $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType);
1972       }
1973     }
1974     if ($errors) {
1975       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', array_filter($errors));
1976     }
1977   }
1978
1979   /**
1980    * get subtypes given the contact type
1981    *
1982    * @param string $contactType
1983    * @return array $subTypes
1984    */
1985   protected function getSubtypes($contactType) {
1986     $subTypes = [];
1987     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
1988
1989     if (count($types) > 0) {
1990       foreach ($types as $type) {
1991         $subTypes[] = $type['name'];
1992       }
1993     }
1994     return $subTypes;
1995   }
1996
1997   /**
1998    * Update the status of the import row to reflect the processing outcome.
1999    *
2000    * @param int $id
2001    * @param string $status
2002    * @param string $message
2003    * @param int|null $entityID
2004    *   Optional created entity ID
2005    * @param array $additionalFields
2006    *  Additional fields to be tracked
2007    * @param array $createdContactIDs
2008    *
2009    * @noinspection PhpDocMissingThrowsInspection
2010    * @noinspection PhpUnhandledExceptionInspection
2011    */
2012   protected function setImportStatus(int $id, string $status, string $message = '', ?int $entityID = NULL, $additionalFields = [], $createdContactIDs = []): void {
2013     foreach ($createdContactIDs as $createdContactID) {
2014       // Store any created contacts for post_actions like tag or add to group.
2015       // These are done on a 'per-batch' status in processPorstActions
2016       // so holding in a property is OK.
2017       $this->createdContacts[$createdContactID] = $createdContactID;
2018     }
2019     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields);
2020   }
2021
2022   /**
2023    * Convert any given date string to default date array.
2024    *
2025    * @param array $params
2026    *   Has given date-format.
2027    * @param array $formatted
2028    *   Store formatted date in this array.
2029    * @param int $dateType
2030    *   Type of date.
2031    * @param string $dateParam
2032    *   Index of params.
2033    */
2034   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2035     //fix for CRM-2687
2036     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
2037     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
2038   }
2039
2040   /**
2041    * Get the value to use for option comparison purposes.
2042    *
2043    * We do a case-insensitive comparison, also swapping ’ for '
2044    * which has at least one known usage (Côte d’Ivoire).
2045    *
2046    * Note we do this to both sides of the comparison.
2047    *
2048    * @param int|string|false|null $importedValue
2049    *
2050    * @return false|int|string|null
2051    */
2052   protected function getComparisonValue($importedValue) {
2053     return is_numeric($importedValue) ? $importedValue : mb_strtolower(str_replace('’', "'", $importedValue));
2054   }
2055
2056 }