CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\UserJob;
  13
  14 /**
  15  *
  16  * @package CRM
  17  * @copyright CiviCRM LLC https://civicrm.org/licensing
  18  */
  19 abstract class CRM_Import_Parser {
  20   /**
  21    * Settings
  22    */
  23   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  24
  25   /**
  26    * Return codes
  27    */
  28   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  29
  30   /**
  31    * Parser modes
  32    */
  33   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  34
  35   /**
  36    * Codes for duplicate record handling
  37    */
  38   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  39
  40   /**
  41    * Contact types
  42    */
  43   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  44
  45
  46   /**
  47    * User job id.
  48    *
  49    * This is the primary key of the civicrm_user_job table which is used to
  50    * track the import.
  51    *
  52    * @var int
  53    */
  54   protected $userJobID;
  55
  56   /**
  57    * @return int|null
  58    */
  59   public function getUserJobID(): ?int {
  60     return $this->userJobID;
  61   }
  62
  63   /**
  64    * Set user job ID.
  65    *
  66    * @param int $userJobID
  67    *
  68    * @return self
  69    */
  70   public function setUserJobID(int $userJobID): self {
  71     $this->userJobID = $userJobID;
  72     return $this;
  73   }
  74
  75   /**
  76    * Get User Job.
  77    *
  78    * API call to retrieve the userJob row.
  79    *
  80    * @return array
  81    *
  82    * @throws \API_Exception
  83    */
  84   protected function getUserJob(): array {
  85     return UserJob::get()
  86       ->addWhere('id', '=', $this->getUserJobID())
  87       ->execute()
  88       ->first();
  89   }
  90
  91   /**
  92    * Get the relevant datasource object.
  93    *
  94    * @return \CRM_Import_DataSource|null
  95    *
  96    * @throws \API_Exception
  97    */
  98   protected function getDataSourceObject(): ?CRM_Import_DataSource {
  99     $className = $this->getSubmittedValue('dataSource');
 100     if ($className) {
 101       /* @var CRM_Import_DataSource $dataSource */
 102       return new $className($this->getUserJobID());
 103     }
 104     return NULL;
 105   }
 106
 107   /**
 108    * Get the submitted value, as stored on the user job.
 109    *
 110    * @param string $fieldName
 111    *
 112    * @return mixed
 113    *
 114    * @throws \API_Exception
 115    */
 116   protected function getSubmittedValue(string $fieldName) {
 117     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 118   }
 119
 120   /**
 121    * Has the import completed.
 122    *
 123    * @return bool
 124    *
 125    * @throws \API_Exception
 126    * @throws \CRM_Core_Exception
 127    */
 128   public function isComplete() :bool {
 129     return $this->getDataSourceObject()->isCompleted();
 130   }
 131
 132   /**
 133    * Get configured contact type.
 134    *
 135    * @throws \API_Exception
 136    */
 137   protected function getContactType() {
 138     if (!$this->_contactType) {
 139       $contactTypeMapping = [
 140         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 141         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 142         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 143       ];
 144       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 145     }
 146     return $this->_contactType;
 147   }
 148
 149   /**
 150    * Get configured contact type.
 151    *
 152    * @return string|null
 153    *
 154    * @throws \API_Exception
 155    */
 156   public function getContactSubType() {
 157     if (!$this->_contactSubType) {
 158       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 159     }
 160     return $this->_contactSubType;
 161   }
 162
 163   /**
 164    * Total number of non empty lines
 165    * @var int
 166    */
 167   protected $_totalCount;
 168
 169   /**
 170    * Running total number of valid lines
 171    * @var int
 172    */
 173   protected $_validCount;
 174
 175   /**
 176    * Running total number of invalid rows
 177    * @var int
 178    */
 179   protected $_invalidRowCount;
 180
 181   /**
 182    * Maximum number of non-empty/comment lines to process
 183    *
 184    * @var int
 185    */
 186   protected $_maxLinesToProcess;
 187
 188   /**
 189    * Array of error lines, bounded by MAX_ERROR
 190    * @var array
 191    */
 192   protected $_errors;
 193
 194   /**
 195    * Total number of duplicate (from database) lines
 196    * @var int
 197    */
 198   protected $_duplicateCount;
 199
 200   /**
 201    * Array of duplicate lines
 202    * @var array
 203    */
 204   protected $_duplicates;
 205
 206   /**
 207    * Maximum number of warnings to store
 208    * @var int
 209    */
 210   protected $_maxWarningCount = self::MAX_WARNINGS;
 211
 212   /**
 213    * Array of warning lines, bounded by MAX_WARNING
 214    * @var array
 215    */
 216   protected $_warnings;
 217
 218   /**
 219    * Array of all the fields that could potentially be part
 220    * of this import process
 221    * @var array
 222    */
 223   protected $_fields;
 224
 225   /**
 226    * Metadata for all available fields, keyed by unique name.
 227    *
 228    * This is intended to supercede $_fields which uses a special sauce format which
 229    * importableFieldsMetadata uses the standard getfields type format.
 230    *
 231    * @var array
 232    */
 233   protected $importableFieldsMetadata = [];
 234
 235   /**
 236    * Get metadata for all importable fields in std getfields style format.
 237    *
 238    * @return array
 239    */
 240   public function getImportableFieldsMetadata(): array {
 241     return $this->importableFieldsMetadata;
 242   }
 243
 244   /**
 245    * Set metadata for all importable fields in std getfields style format.
 246    *
 247    * @param array $importableFieldsMetadata
 248    */
 249   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 250     $this->importableFieldsMetadata = $importableFieldsMetadata;
 251   }
 252
 253   /**
 254    * Array of the fields that are actually part of the import process
 255    * the position in the array also dictates their position in the import
 256    * file
 257    * @var array
 258    */
 259   protected $_activeFields;
 260
 261   /**
 262    * Cache the count of active fields
 263    *
 264    * @var int
 265    */
 266   protected $_activeFieldCount;
 267
 268   /**
 269    * Cache of preview rows
 270    *
 271    * @var array
 272    */
 273   protected $_rows;
 274
 275   /**
 276    * Filename of error data
 277    *
 278    * @var string
 279    */
 280   protected $_errorFileName;
 281
 282   /**
 283    * Filename of duplicate data
 284    *
 285    * @var string
 286    */
 287   protected $_duplicateFileName;
 288
 289   /**
 290    * Contact type
 291    *
 292    * @var string
 293    */
 294   public $_contactType;
 295
 296   /**
 297    * @param string $contactType
 298    *
 299    * @return CRM_Import_Parser
 300    */
 301   public function setContactType(string $contactType): CRM_Import_Parser {
 302     $this->_contactType = $contactType;
 303     return $this;
 304   }
 305
 306   /**
 307    * Contact sub-type
 308    *
 309    * @var int|null
 310    */
 311   public $_contactSubType;
 312
 313   /**
 314    * @param int|null $contactSubType
 315    *
 316    * @return self
 317    */
 318   public function setContactSubType(?int $contactSubType): self {
 319     $this->_contactSubType = $contactSubType;
 320     return $this;
 321   }
 322
 323   /**
 324    * Class constructor.
 325    */
 326   public function __construct() {
 327     $this->_maxLinesToProcess = 0;
 328   }
 329
 330   /**
 331    * Set and validate field values.
 332    *
 333    * @param array $elements
 334    *   array.
 335    */
 336   public function setActiveFieldValues($elements): void {
 337     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 338     for ($i = 0; $i < $maxCount; $i++) {
 339       $this->_activeFields[$i]->setValue($elements[$i]);
 340     }
 341
 342     // reset all the values that we did not have an equivalent import element
 343     for (; $i < $this->_activeFieldCount; $i++) {
 344       $this->_activeFields[$i]->resetValue();
 345     }
 346   }
 347
 348   /**
 349    * Format the field values for input to the api.
 350    *
 351    * @return array
 352    *   (reference) associative array of name/value pairs
 353    */
 354   public function &getActiveFieldParams() {
 355     $params = [];
 356     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 357       if (isset($this->_activeFields[$i]->_value)
 358         && !isset($params[$this->_activeFields[$i]->_name])
 359         && !isset($this->_activeFields[$i]->_related)
 360       ) {
 361
 362         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 363       }
 364     }
 365     return $params;
 366   }
 367
 368   /**
 369    * Add progress bar to the import process. Calculates time remaining, status etc.
 370    *
 371    * @param $statusID
 372    *   status id of the import process saved in $config->uploadDir.
 373    * @param bool $startImport
 374    *   True when progress bar is to be initiated.
 375    * @param $startTimestamp
 376    *   Initial timestamp when the import was started.
 377    * @param $prevTimestamp
 378    *   Previous timestamp when this function was last called.
 379    * @param $totalRowCount
 380    *   Total number of rows in the import file.
 381    *
 382    * @return NULL|$currTimestamp
 383    */
 384   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 385     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 386
 387     if ($startImport) {
 388       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 389       //do not force the browser to display the save dialog, CRM-7640
 390       $contents = json_encode([0, $status]);
 391       file_put_contents($statusFile, $contents);
 392     }
 393     else {
 394       $rowCount = $this->_rowCount ?? $this->_lineCount;
 395       $currTimestamp = time();
 396       $time = ($currTimestamp - $prevTimestamp);
 397       $recordsLeft = $totalRowCount - $rowCount;
 398       if ($recordsLeft < 0) {
 399         $recordsLeft = 0;
 400       }
 401       $estimatedTime = ($recordsLeft / 50) * $time;
 402       $estMinutes = floor($estimatedTime / 60);
 403       $timeFormatted = '';
 404       if ($estMinutes > 1) {
 405         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 406         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 407       }
 408       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 409       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 410       $statusMsg = ts('%1 of %2 records - %3 remaining',
 411         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 412       );
 413       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 414       $contents = json_encode([$processedPercent, $status]);
 415
 416       file_put_contents($statusFile, $contents);
 417       return $currTimestamp;
 418     }
 419   }
 420
 421   /**
 422    * @return array
 423    */
 424   public function getSelectValues(): array {
 425     $values = [];
 426     foreach ($this->_fields as $name => $field) {
 427       $values[$name] = $field->_title;
 428     }
 429     return $values;
 430   }
 431
 432   /**
 433    * @return array
 434    */
 435   public function getSelectTypes() {
 436     $values = [];
 437     // This is only called from the MapField form in isolation now,
 438     // so we need to set the metadata.
 439     $this->init();
 440     foreach ($this->_fields as $name => $field) {
 441       if (isset($field->_hasLocationType)) {
 442         $values[$name] = $field->_hasLocationType;
 443       }
 444     }
 445     return $values;
 446   }
 447
 448   /**
 449    * @return array
 450    */
 451   public function getHeaderPatterns() {
 452     $values = [];
 453     foreach ($this->_fields as $name => $field) {
 454       if (isset($field->_headerPattern)) {
 455         $values[$name] = $field->_headerPattern;
 456       }
 457     }
 458     return $values;
 459   }
 460
 461   /**
 462    * @return array
 463    */
 464   public function getDataPatterns() {
 465     $values = [];
 466     foreach ($this->_fields as $name => $field) {
 467       $values[$name] = $field->_dataPattern;
 468     }
 469     return $values;
 470   }
 471
 472   /**
 473    * Remove single-quote enclosures from a value array (row).
 474    *
 475    * @param array $values
 476    * @param string $enclosure
 477    *
 478    * @return void
 479    */
 480   public static function encloseScrub(&$values, $enclosure = "'") {
 481     if (empty($values)) {
 482       return;
 483     }
 484
 485     foreach ($values as $k => $v) {
 486       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 487     }
 488   }
 489
 490   /**
 491    * Setter function.
 492    *
 493    * @param int $max
 494    *
 495    * @return void
 496    */
 497   public function setMaxLinesToProcess($max) {
 498     $this->_maxLinesToProcess = $max;
 499   }
 500
 501   /**
 502    * Determines the file extension based on error code.
 503    *
 504    * @var int $type error code constant
 505    * @return string
 506    */
 507   public static function errorFileName($type) {
 508     $fileName = NULL;
 509     if (empty($type)) {
 510       return $fileName;
 511     }
 512
 513     $config = CRM_Core_Config::singleton();
 514     $fileName = $config->uploadDir . "sqlImport";
 515     switch ($type) {
 516       case self::ERROR:
 517         $fileName .= '.errors';
 518         break;
 519
 520       case self::DUPLICATE:
 521         $fileName .= '.duplicates';
 522         break;
 523
 524       case self::NO_MATCH:
 525         $fileName .= '.mismatch';
 526         break;
 527
 528       case self::UNPARSED_ADDRESS_WARNING:
 529         $fileName .= '.unparsedAddress';
 530         break;
 531     }
 532
 533     return $fileName;
 534   }
 535
 536   /**
 537    * Determines the file name based on error code.
 538    *
 539    * @var $type error code constant
 540    * @return string
 541    */
 542   public static function saveFileName($type) {
 543     $fileName = NULL;
 544     if (empty($type)) {
 545       return $fileName;
 546     }
 547     switch ($type) {
 548       case self::ERROR:
 549         $fileName = 'Import_Errors.csv';
 550         break;
 551
 552       case self::DUPLICATE:
 553         $fileName = 'Import_Duplicates.csv';
 554         break;
 555
 556       case self::NO_MATCH:
 557         $fileName = 'Import_Mismatch.csv';
 558         break;
 559
 560       case self::UNPARSED_ADDRESS_WARNING:
 561         $fileName = 'Import_Unparsed_Address.csv';
 562         break;
 563     }
 564
 565     return $fileName;
 566   }
 567
 568   /**
 569    * Check if contact is a duplicate .
 570    *
 571    * @param array $formatValues
 572    *
 573    * @return array
 574    */
 575   protected function checkContactDuplicate(&$formatValues) {
 576     //retrieve contact id using contact dedupe rule
 577     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 578     $formatValues['version'] = 3;
 579     require_once 'CRM/Utils/DeprecatedUtils.php';
 580     $params = $formatValues;
 581     static $cIndieFields = NULL;
 582     static $defaultLocationId = NULL;
 583
 584     $contactType = $params['contact_type'];
 585     if ($cIndieFields == NULL) {
 586       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 587       $cIndieFields = $cTempIndieFields;
 588
 589       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 590
 591       // set the value to default location id else set to 1
 592       if (!$defaultLocationId = (int) $defaultLocation->id) {
 593         $defaultLocationId = 1;
 594       }
 595     }
 596
 597     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 598
 599     $contactFormatted = [];
 600     foreach ($params as $key => $field) {
 601       if ($field == NULL || $field === '') {
 602         continue;
 603       }
 604       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 605       // instead of soft credit contact.
 606       if (is_array($field) && $key != "soft_credit") {
 607         foreach ($field as $value) {
 608           $break = FALSE;
 609           if (is_array($value)) {
 610             foreach ($value as $name => $testForEmpty) {
 611               if ($name !== 'phone_type' &&
 612                 ($testForEmpty === '' || $testForEmpty == NULL)
 613               ) {
 614                 $break = TRUE;
 615                 break;
 616               }
 617             }
 618           }
 619           else {
 620             $break = TRUE;
 621           }
 622           if (!$break) {
 623             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 624           }
 625         }
 626         continue;
 627       }
 628
 629       $value = [$key => $field];
 630
 631       // check if location related field, then we need to add primary location type
 632       if (in_array($key, $locationFields)) {
 633         $value['location_type_id'] = $defaultLocationId;
 634       }
 635       elseif (array_key_exists($key, $cIndieFields)) {
 636         $value['contact_type'] = $contactType;
 637       }
 638
 639       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 640     }
 641
 642     $contactFormatted['contact_type'] = $contactType;
 643
 644     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 645   }
 646
 647   /**
 648    * This function adds the contact variable in $values to the
 649    * parameter list $params.  For most cases, $values should have length 1.  If
 650    * the variable being added is a child of Location, a location_type_id must
 651    * also be included.  If it is a child of phone, a phone_type must be included.
 652    *
 653    * @param array $values
 654    *   The variable(s) to be added.
 655    * @param array $params
 656    *   The structured parameter list.
 657    *
 658    * @return bool|CRM_Utils_Error
 659    */
 660   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 661     // @todo - like most functions in import ... most of this is cruft....
 662     // Crawl through the possible classes:
 663     // Contact
 664     //      Individual
 665     //      Household
 666     //      Organization
 667     //          Location
 668     //              Address
 669     //              Email
 670     //              Phone
 671     //              IM
 672     //      Note
 673     //      Custom
 674
 675     // Cache the various object fields
 676     static $fields = NULL;
 677
 678     if ($fields == NULL) {
 679       $fields = [];
 680     }
 681
 682     // first add core contact values since for other Civi modules they are not added
 683     require_once 'CRM/Contact/BAO/Contact.php';
 684     $contactFields = CRM_Contact_DAO_Contact::fields();
 685     _civicrm_api3_store_values($contactFields, $values, $params);
 686
 687     if (isset($values['contact_type'])) {
 688       // we're an individual/household/org property
 689
 690       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 691
 692       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 693       return TRUE;
 694     }
 695
 696     if (isset($values['individual_prefix'])) {
 697       if (!empty($params['prefix_id'])) {
 698         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 699         $params['prefix'] = $prefixes[$params['prefix_id']];
 700       }
 701       else {
 702         $params['prefix'] = $values['individual_prefix'];
 703       }
 704       return TRUE;
 705     }
 706
 707     if (isset($values['individual_suffix'])) {
 708       if (!empty($params['suffix_id'])) {
 709         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 710         $params['suffix'] = $suffixes[$params['suffix_id']];
 711       }
 712       else {
 713         $params['suffix'] = $values['individual_suffix'];
 714       }
 715       return TRUE;
 716     }
 717
 718     // CRM-4575
 719     if (isset($values['email_greeting'])) {
 720       if (!empty($params['email_greeting_id'])) {
 721         $emailGreetingFilter = [
 722           'contact_type' => $params['contact_type'] ?? NULL,
 723           'greeting_type' => 'email_greeting',
 724         ];
 725         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
 726         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
 727       }
 728       else {
 729         $params['email_greeting'] = $values['email_greeting'];
 730       }
 731
 732       return TRUE;
 733     }
 734
 735     if (isset($values['postal_greeting'])) {
 736       if (!empty($params['postal_greeting_id'])) {
 737         $postalGreetingFilter = [
 738           'contact_type' => $params['contact_type'] ?? NULL,
 739           'greeting_type' => 'postal_greeting',
 740         ];
 741         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
 742         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
 743       }
 744       else {
 745         $params['postal_greeting'] = $values['postal_greeting'];
 746       }
 747       return TRUE;
 748     }
 749
 750     if (isset($values['addressee'])) {
 751       $params['addressee'] = $values['addressee'];
 752       return TRUE;
 753     }
 754
 755     if (isset($values['gender'])) {
 756       if (!empty($params['gender_id'])) {
 757         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 758         $params['gender'] = $genders[$params['gender_id']];
 759       }
 760       else {
 761         $params['gender'] = $values['gender'];
 762       }
 763       return TRUE;
 764     }
 765
 766     if (!empty($values['preferred_communication_method'])) {
 767       $comm = [];
 768       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
 769
 770       $preffComm = explode(',', $values['preferred_communication_method']);
 771       foreach ($preffComm as $v) {
 772         $v = strtolower(trim($v));
 773         if (array_key_exists($v, $pcm)) {
 774           $comm[$pcm[$v]] = 1;
 775         }
 776       }
 777
 778       $params['preferred_communication_method'] = $comm;
 779       return TRUE;
 780     }
 781
 782     // format the website params.
 783     if (!empty($values['url'])) {
 784       static $websiteFields;
 785       if (!is_array($websiteFields)) {
 786         require_once 'CRM/Core/DAO/Website.php';
 787         $websiteFields = CRM_Core_DAO_Website::fields();
 788       }
 789       if (!array_key_exists('website', $params) ||
 790         !is_array($params['website'])
 791       ) {
 792         $params['website'] = [];
 793       }
 794
 795       $websiteCount = count($params['website']);
 796       _civicrm_api3_store_values($websiteFields, $values,
 797         $params['website'][++$websiteCount]
 798       );
 799
 800       return TRUE;
 801     }
 802
 803     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 804     if (!empty($values['location_type_id'])) {
 805       static $fields = NULL;
 806       if ($fields == NULL) {
 807         $fields = [];
 808       }
 809
 810       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 811         $name = strtolower($block);
 812         if (!array_key_exists($name, $values)) {
 813           continue;
 814         }
 815
 816         if ($name === 'phone_ext') {
 817           $block = 'Phone';
 818         }
 819
 820         // block present in value array.
 821         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 822           $params[$name] = [];
 823         }
 824
 825         if (!array_key_exists($block, $fields)) {
 826           $className = "CRM_Core_DAO_$block";
 827           $fields[$block] =& $className::fields();
 828         }
 829
 830         $blockCnt = count($params[$name]);
 831
 832         // copy value to dao field name.
 833         if ($name == 'im') {
 834           $values['name'] = $values[$name];
 835         }
 836
 837         _civicrm_api3_store_values($fields[$block], $values,
 838           $params[$name][++$blockCnt]
 839         );
 840
 841         if (empty($params['id']) && ($blockCnt == 1)) {
 842           $params[$name][$blockCnt]['is_primary'] = TRUE;
 843         }
 844
 845         // we only process single block at a time.
 846         return TRUE;
 847       }
 848
 849       // handle address fields.
 850       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
 851         $params['address'] = [];
 852       }
 853
 854       $addressCnt = 1;
 855       foreach ($params['address'] as $cnt => $addressBlock) {
 856         if (CRM_Utils_Array::value('location_type_id', $values) ==
 857           CRM_Utils_Array::value('location_type_id', $addressBlock)
 858         ) {
 859           $addressCnt = $cnt;
 860           break;
 861         }
 862         $addressCnt++;
 863       }
 864
 865       if (!array_key_exists('Address', $fields)) {
 866         $fields['Address'] = CRM_Core_DAO_Address::fields();
 867       }
 868
 869       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
 870       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
 871       // the address in CRM_Core_BAO_Address::create method
 872       if (!empty($values['location_type_id'])) {
 873         static $customFields = [];
 874         if (empty($customFields)) {
 875           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
 876         }
 877         // make a copy of values, as we going to make changes
 878         $newValues = $values;
 879         foreach ($values as $key => $val) {
 880           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 881           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
 882             // mark an entry in fields array since we want the value of custom field to be copied
 883             $fields['Address'][$key] = NULL;
 884
 885             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 886             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
 887               $mulValues = explode(',', $val);
 888               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 889               $newValues[$key] = [];
 890               foreach ($mulValues as $v1) {
 891                 foreach ($customOption as $v2) {
 892                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 893                     (strtolower($v2['value']) == strtolower(trim($v1)))
 894                   ) {
 895                     if ($htmlType == 'CheckBox') {
 896                       $newValues[$key][$v2['value']] = 1;
 897                     }
 898                     else {
 899                       $newValues[$key][] = $v2['value'];
 900                     }
 901                   }
 902                 }
 903               }
 904             }
 905           }
 906         }
 907         // consider new values
 908         $values = $newValues;
 909       }
 910
 911       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
 912
 913       $addressFields = [
 914         'county',
 915         'country',
 916         'state_province',
 917         'supplemental_address_1',
 918         'supplemental_address_2',
 919         'supplemental_address_3',
 920         'StateProvince.name',
 921       ];
 922
 923       foreach ($addressFields as $field) {
 924         if (array_key_exists($field, $values)) {
 925           if (!array_key_exists('address', $params)) {
 926             $params['address'] = [];
 927           }
 928           $params['address'][$addressCnt][$field] = $values[$field];
 929         }
 930       }
 931
 932       if ($addressCnt == 1) {
 933
 934         $params['address'][$addressCnt]['is_primary'] = TRUE;
 935       }
 936       return TRUE;
 937     }
 938
 939     if (isset($values['note'])) {
 940       // add a note field
 941       if (!isset($params['note'])) {
 942         $params['note'] = [];
 943       }
 944       $noteBlock = count($params['note']) + 1;
 945
 946       $params['note'][$noteBlock] = [];
 947       if (!isset($fields['Note'])) {
 948         $fields['Note'] = CRM_Core_DAO_Note::fields();
 949       }
 950
 951       // get the current logged in civicrm user
 952       $session = CRM_Core_Session::singleton();
 953       $userID = $session->get('userID');
 954
 955       if ($userID) {
 956         $values['contact_id'] = $userID;
 957       }
 958
 959       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
 960
 961       return TRUE;
 962     }
 963
 964     // Check for custom field values
 965
 966     if (empty($fields['custom'])) {
 967       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
 968         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
 969       );
 970     }
 971
 972     foreach ($values as $key => $value) {
 973       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 974         // check if it's a valid custom field id
 975
 976         if (!array_key_exists($customFieldID, $fields['custom'])) {
 977           return civicrm_api3_create_error('Invalid custom field ID');
 978         }
 979         else {
 980           $params[$key] = $value;
 981         }
 982       }
 983     }
 984   }
 985
 986   /**
 987    * Parse a field which could be represented by a label or name value rather than the DB value.
 988    *
 989    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
 990    *
 991    * but if not available then see if we have a label that can be converted to a name.
 992    *
 993    * @param string|int|null $submittedValue
 994    * @param array $fieldSpec
 995    *   Metadata for the field
 996    *
 997    * @return mixed
 998    */
 999   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1000     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1001     if (!isset($fieldSpec['bao'])) {
1002       return $submittedValue;
1003     }
1004     /* @var \CRM_Core_DAO $bao */
1005     $bao = $fieldSpec['bao'];
1006     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1007     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1008     if (isset($nameOptions[$submittedValue])) {
1009       return $submittedValue;
1010     }
1011     if (in_array($submittedValue, $nameOptions)) {
1012       return array_search($submittedValue, $nameOptions, TRUE);
1013     }
1014
1015     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1016     if (isset($labelOptions[$submittedValue])) {
1017       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1018     }
1019     return '';
1020   }
1021
1022   /**
1023    * This is code extracted from 4 places where this exact snippet was being duplicated.
1024    *
1025    * FIXME: Extracting this was a first step, but there's also
1026    *  1. Inconsistency in the way other select options are handled.
1027    *     Contribution adds handling for Select/Radio/Autocomplete
1028    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1029    *     Membership is missing all of it
1030    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1031    *
1032    * @param $customFieldID
1033    * @param $value
1034    * @param $fieldType
1035    * @return array
1036    */
1037   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1038     $mulValues = explode(',', $value);
1039     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1040     $values = [];
1041     foreach ($mulValues as $v1) {
1042       foreach ($customOption as $customValueID => $customLabel) {
1043         $customValue = $customLabel['value'];
1044         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1045           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1046         ) {
1047           $values[] = $customValue;
1048         }
1049       }
1050     }
1051     return $values;
1052   }
1053
1054   /**
1055    * Get the ids of any contacts that match according to the rule.
1056    *
1057    * @param array $formatted
1058    *
1059    * @return array
1060    */
1061   protected function getIdsOfMatchingContacts(array $formatted):array {
1062     // the call to the deprecated function seems to add no value other that to do an additional
1063     // check for the contact_id & type.
1064     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1065     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
1066       return [];
1067     }
1068     if (is_array($error['error_message']['params'][0])) {
1069       return $error['error_message']['params'][0];
1070     }
1071     else {
1072       return explode(',', $error['error_message']['params'][0]);
1073     }
1074   }
1075
1076 }