CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\UserJob;
  13
  14 /**
  15  *
  16  * @package CRM
  17  * @copyright CiviCRM LLC https://civicrm.org/licensing
  18  */
  19 abstract class CRM_Import_Parser {
  20   /**
  21    * Settings
  22    */
  23   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  24
  25   /**
  26    * Return codes
  27    */
  28   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  29
  30   /**
  31    * Parser modes
  32    */
  33   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  34
  35   /**
  36    * Codes for duplicate record handling
  37    */
  38   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  39
  40   /**
  41    * Contact types
  42    */
  43   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  44
  45
  46   /**
  47    * User job id.
  48    *
  49    * This is the primary key of the civicrm_user_job table which is used to
  50    * track the import.
  51    *
  52    * @var int
  53    */
  54   protected $userJobID;
  55
  56   /**
  57    * @return int|null
  58    */
  59   public function getUserJobID(): ?int {
  60     return $this->userJobID;
  61   }
  62
  63   /**
  64    * Set user job ID.
  65    *
  66    * @param int $userJobID
  67    */
  68   public function setUserJobID(int $userJobID): void {
  69     $this->userJobID = $userJobID;
  70   }
  71
  72   /**
  73    * Get User Job.
  74    *
  75    * API call to retrieve the userJob row.
  76    *
  77    * @return array
  78    *
  79    * @throws \API_Exception
  80    */
  81   protected function getUserJob(): array {
  82     return UserJob::get()
  83       ->addWhere('id', '=', $this->getUserJobID())
  84       ->execute()
  85       ->first();
  86   }
  87
  88   /**
  89    * Get the submitted value, as stored on the user job.
  90    *
  91    * @param string $fieldName
  92    *
  93    * @return mixed
  94    *
  95    * @throws \API_Exception
  96    */
  97   protected function getSubmittedValue(string $fieldName) {
  98     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
  99   }
 100
 101   /**
 102    * Get configured contact type.
 103    *
 104    * @throws \API_Exception
 105    */
 106   protected function getContactType() {
 107     if (!$this->_contactType) {
 108       $contactTypeMapping = [
 109         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 110         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 111         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 112       ];
 113       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 114     }
 115     return $this->_contactType;
 116   }
 117
 118   /**
 119    * Get configured contact type.
 120    *
 121    * @return string|null
 122    *
 123    * @throws \API_Exception
 124    */
 125   public function getContactSubType() {
 126     if (!$this->_contactSubType) {
 127       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 128     }
 129     return $this->_contactSubType;
 130   }
 131
 132   /**
 133    * Total number of non empty lines
 134    * @var int
 135    */
 136   protected $_totalCount;
 137
 138   /**
 139    * Running total number of valid lines
 140    * @var int
 141    */
 142   protected $_validCount;
 143
 144   /**
 145    * Running total number of invalid rows
 146    * @var int
 147    */
 148   protected $_invalidRowCount;
 149
 150   /**
 151    * Maximum number of non-empty/comment lines to process
 152    *
 153    * @var int
 154    */
 155   protected $_maxLinesToProcess;
 156
 157   /**
 158    * Array of error lines, bounded by MAX_ERROR
 159    * @var array
 160    */
 161   protected $_errors;
 162
 163   /**
 164    * Total number of conflict lines
 165    * @var int
 166    */
 167   protected $_conflictCount;
 168
 169   /**
 170    * Array of conflict lines
 171    * @var array
 172    */
 173   protected $_conflicts;
 174
 175   /**
 176    * Total number of duplicate (from database) lines
 177    * @var int
 178    */
 179   protected $_duplicateCount;
 180
 181   /**
 182    * Array of duplicate lines
 183    * @var array
 184    */
 185   protected $_duplicates;
 186
 187   /**
 188    * Maximum number of warnings to store
 189    * @var int
 190    */
 191   protected $_maxWarningCount = self::MAX_WARNINGS;
 192
 193   /**
 194    * Array of warning lines, bounded by MAX_WARNING
 195    * @var array
 196    */
 197   protected $_warnings;
 198
 199   /**
 200    * Array of all the fields that could potentially be part
 201    * of this import process
 202    * @var array
 203    */
 204   protected $_fields;
 205
 206   /**
 207    * Metadata for all available fields, keyed by unique name.
 208    *
 209    * This is intended to supercede $_fields which uses a special sauce format which
 210    * importableFieldsMetadata uses the standard getfields type format.
 211    *
 212    * @var array
 213    */
 214   protected $importableFieldsMetadata = [];
 215
 216   /**
 217    * Get metadata for all importable fields in std getfields style format.
 218    *
 219    * @return array
 220    */
 221   public function getImportableFieldsMetadata(): array {
 222     return $this->importableFieldsMetadata;
 223   }
 224
 225   /**
 226    * Set metadata for all importable fields in std getfields style format.
 227    *
 228    * @param array $importableFieldsMetadata
 229    */
 230   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 231     $this->importableFieldsMetadata = $importableFieldsMetadata;
 232   }
 233
 234   /**
 235    * Array of the fields that are actually part of the import process
 236    * the position in the array also dictates their position in the import
 237    * file
 238    * @var array
 239    */
 240   protected $_activeFields;
 241
 242   /**
 243    * Cache the count of active fields
 244    *
 245    * @var int
 246    */
 247   protected $_activeFieldCount;
 248
 249   /**
 250    * Cache of preview rows
 251    *
 252    * @var array
 253    */
 254   protected $_rows;
 255
 256   /**
 257    * Filename of error data
 258    *
 259    * @var string
 260    */
 261   protected $_errorFileName;
 262
 263   /**
 264    * Filename of conflict data
 265    *
 266    * @var string
 267    */
 268   protected $_conflictFileName;
 269
 270   /**
 271    * Filename of duplicate data
 272    *
 273    * @var string
 274    */
 275   protected $_duplicateFileName;
 276
 277   /**
 278    * Contact type
 279    *
 280    * @var string
 281    */
 282   public $_contactType;
 283
 284   /**
 285    * @param string $contactType
 286    *
 287    * @return CRM_Import_Parser
 288    */
 289   public function setContactType(string $contactType): CRM_Import_Parser {
 290     $this->_contactType = $contactType;
 291     return $this;
 292   }
 293
 294   /**
 295    * Contact sub-type
 296    *
 297    * @var int|null
 298    */
 299   public $_contactSubType;
 300
 301   /**
 302    * @param int|null $contactSubType
 303    *
 304    * @return self
 305    */
 306   public function setContactSubType(?int $contactSubType): self {
 307     $this->_contactSubType = $contactSubType;
 308     return $this;
 309   }
 310
 311   /**
 312    * Class constructor.
 313    */
 314   public function __construct() {
 315     $this->_maxLinesToProcess = 0;
 316   }
 317
 318   /**
 319    * Set and validate field values.
 320    *
 321    * @param array $elements
 322    *   array.
 323    */
 324   public function setActiveFieldValues($elements): void {
 325     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 326     for ($i = 0; $i < $maxCount; $i++) {
 327       $this->_activeFields[$i]->setValue($elements[$i]);
 328     }
 329
 330     // reset all the values that we did not have an equivalent import element
 331     for (; $i < $this->_activeFieldCount; $i++) {
 332       $this->_activeFields[$i]->resetValue();
 333     }
 334   }
 335
 336   /**
 337    * Format the field values for input to the api.
 338    *
 339    * @return array
 340    *   (reference) associative array of name/value pairs
 341    */
 342   public function &getActiveFieldParams() {
 343     $params = [];
 344     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 345       if (isset($this->_activeFields[$i]->_value)
 346         && !isset($params[$this->_activeFields[$i]->_name])
 347         && !isset($this->_activeFields[$i]->_related)
 348       ) {
 349
 350         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 351       }
 352     }
 353     return $params;
 354   }
 355
 356   /**
 357    * Add progress bar to the import process. Calculates time remaining, status etc.
 358    *
 359    * @param $statusID
 360    *   status id of the import process saved in $config->uploadDir.
 361    * @param bool $startImport
 362    *   True when progress bar is to be initiated.
 363    * @param $startTimestamp
 364    *   Initial timestamp when the import was started.
 365    * @param $prevTimestamp
 366    *   Previous timestamp when this function was last called.
 367    * @param $totalRowCount
 368    *   Total number of rows in the import file.
 369    *
 370    * @return NULL|$currTimestamp
 371    */
 372   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 373     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 374
 375     if ($startImport) {
 376       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 377       //do not force the browser to display the save dialog, CRM-7640
 378       $contents = json_encode([0, $status]);
 379       file_put_contents($statusFile, $contents);
 380     }
 381     else {
 382       $rowCount = $this->_rowCount ?? $this->_lineCount;
 383       $currTimestamp = time();
 384       $time = ($currTimestamp - $prevTimestamp);
 385       $recordsLeft = $totalRowCount - $rowCount;
 386       if ($recordsLeft < 0) {
 387         $recordsLeft = 0;
 388       }
 389       $estimatedTime = ($recordsLeft / 50) * $time;
 390       $estMinutes = floor($estimatedTime / 60);
 391       $timeFormatted = '';
 392       if ($estMinutes > 1) {
 393         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 394         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 395       }
 396       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 397       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 398       $statusMsg = ts('%1 of %2 records - %3 remaining',
 399         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 400       );
 401       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 402       $contents = json_encode([$processedPercent, $status]);
 403
 404       file_put_contents($statusFile, $contents);
 405       return $currTimestamp;
 406     }
 407   }
 408
 409   /**
 410    * @return array
 411    */
 412   public function getSelectValues(): array {
 413     $values = [];
 414     foreach ($this->_fields as $name => $field) {
 415       $values[$name] = $field->_title;
 416     }
 417     return $values;
 418   }
 419
 420   /**
 421    * @return array
 422    */
 423   public function getSelectTypes() {
 424     $values = [];
 425     foreach ($this->_fields as $name => $field) {
 426       if (isset($field->_hasLocationType)) {
 427         $values[$name] = $field->_hasLocationType;
 428       }
 429     }
 430     return $values;
 431   }
 432
 433   /**
 434    * @return array
 435    */
 436   public function getHeaderPatterns() {
 437     $values = [];
 438     foreach ($this->_fields as $name => $field) {
 439       if (isset($field->_headerPattern)) {
 440         $values[$name] = $field->_headerPattern;
 441       }
 442     }
 443     return $values;
 444   }
 445
 446   /**
 447    * @return array
 448    */
 449   public function getDataPatterns() {
 450     $values = [];
 451     foreach ($this->_fields as $name => $field) {
 452       $values[$name] = $field->_dataPattern;
 453     }
 454     return $values;
 455   }
 456
 457   /**
 458    * Remove single-quote enclosures from a value array (row).
 459    *
 460    * @param array $values
 461    * @param string $enclosure
 462    *
 463    * @return void
 464    */
 465   public static function encloseScrub(&$values, $enclosure = "'") {
 466     if (empty($values)) {
 467       return;
 468     }
 469
 470     foreach ($values as $k => $v) {
 471       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 472     }
 473   }
 474
 475   /**
 476    * Setter function.
 477    *
 478    * @param int $max
 479    *
 480    * @return void
 481    */
 482   public function setMaxLinesToProcess($max) {
 483     $this->_maxLinesToProcess = $max;
 484   }
 485
 486   /**
 487    * Determines the file extension based on error code.
 488    *
 489    * @var int $type error code constant
 490    * @return string
 491    */
 492   public static function errorFileName($type) {
 493     $fileName = NULL;
 494     if (empty($type)) {
 495       return $fileName;
 496     }
 497
 498     $config = CRM_Core_Config::singleton();
 499     $fileName = $config->uploadDir . "sqlImport";
 500     switch ($type) {
 501       case self::ERROR:
 502         $fileName .= '.errors';
 503         break;
 504
 505       case self::CONFLICT:
 506         $fileName .= '.conflicts';
 507         break;
 508
 509       case self::DUPLICATE:
 510         $fileName .= '.duplicates';
 511         break;
 512
 513       case self::NO_MATCH:
 514         $fileName .= '.mismatch';
 515         break;
 516
 517       case self::UNPARSED_ADDRESS_WARNING:
 518         $fileName .= '.unparsedAddress';
 519         break;
 520     }
 521
 522     return $fileName;
 523   }
 524
 525   /**
 526    * Determines the file name based on error code.
 527    *
 528    * @var $type error code constant
 529    * @return string
 530    */
 531   public static function saveFileName($type) {
 532     $fileName = NULL;
 533     if (empty($type)) {
 534       return $fileName;
 535     }
 536     switch ($type) {
 537       case self::ERROR:
 538         $fileName = 'Import_Errors.csv';
 539         break;
 540
 541       case self::CONFLICT:
 542         $fileName = 'Import_Conflicts.csv';
 543         break;
 544
 545       case self::DUPLICATE:
 546         $fileName = 'Import_Duplicates.csv';
 547         break;
 548
 549       case self::NO_MATCH:
 550         $fileName = 'Import_Mismatch.csv';
 551         break;
 552
 553       case self::UNPARSED_ADDRESS_WARNING:
 554         $fileName = 'Import_Unparsed_Address.csv';
 555         break;
 556     }
 557
 558     return $fileName;
 559   }
 560
 561   /**
 562    * Check if contact is a duplicate .
 563    *
 564    * @param array $formatValues
 565    *
 566    * @return array
 567    */
 568   protected function checkContactDuplicate(&$formatValues) {
 569     //retrieve contact id using contact dedupe rule
 570     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 571     $formatValues['version'] = 3;
 572     require_once 'CRM/Utils/DeprecatedUtils.php';
 573     $params = $formatValues;
 574     static $cIndieFields = NULL;
 575     static $defaultLocationId = NULL;
 576
 577     $contactType = $params['contact_type'];
 578     if ($cIndieFields == NULL) {
 579       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 580       $cIndieFields = $cTempIndieFields;
 581
 582       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 583
 584       // set the value to default location id else set to 1
 585       if (!$defaultLocationId = (int) $defaultLocation->id) {
 586         $defaultLocationId = 1;
 587       }
 588     }
 589
 590     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 591
 592     $contactFormatted = [];
 593     foreach ($params as $key => $field) {
 594       if ($field == NULL || $field === '') {
 595         continue;
 596       }
 597       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 598       // instead of soft credit contact.
 599       if (is_array($field) && $key != "soft_credit") {
 600         foreach ($field as $value) {
 601           $break = FALSE;
 602           if (is_array($value)) {
 603             foreach ($value as $name => $testForEmpty) {
 604               if ($name !== 'phone_type' &&
 605                 ($testForEmpty === '' || $testForEmpty == NULL)
 606               ) {
 607                 $break = TRUE;
 608                 break;
 609               }
 610             }
 611           }
 612           else {
 613             $break = TRUE;
 614           }
 615           if (!$break) {
 616             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 617           }
 618         }
 619         continue;
 620       }
 621
 622       $value = [$key => $field];
 623
 624       // check if location related field, then we need to add primary location type
 625       if (in_array($key, $locationFields)) {
 626         $value['location_type_id'] = $defaultLocationId;
 627       }
 628       elseif (array_key_exists($key, $cIndieFields)) {
 629         $value['contact_type'] = $contactType;
 630       }
 631
 632       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 633     }
 634
 635     $contactFormatted['contact_type'] = $contactType;
 636
 637     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 638   }
 639
 640   /**
 641    * This function adds the contact variable in $values to the
 642    * parameter list $params.  For most cases, $values should have length 1.  If
 643    * the variable being added is a child of Location, a location_type_id must
 644    * also be included.  If it is a child of phone, a phone_type must be included.
 645    *
 646    * @param array $values
 647    *   The variable(s) to be added.
 648    * @param array $params
 649    *   The structured parameter list.
 650    *
 651    * @return bool|CRM_Utils_Error
 652    */
 653   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 654     // @todo - like most functions in import ... most of this is cruft....
 655     // Crawl through the possible classes:
 656     // Contact
 657     //      Individual
 658     //      Household
 659     //      Organization
 660     //          Location
 661     //              Address
 662     //              Email
 663     //              Phone
 664     //              IM
 665     //      Note
 666     //      Custom
 667
 668     // Cache the various object fields
 669     static $fields = NULL;
 670
 671     if ($fields == NULL) {
 672       $fields = [];
 673     }
 674
 675     // first add core contact values since for other Civi modules they are not added
 676     require_once 'CRM/Contact/BAO/Contact.php';
 677     $contactFields = CRM_Contact_DAO_Contact::fields();
 678     _civicrm_api3_store_values($contactFields, $values, $params);
 679
 680     if (isset($values['contact_type'])) {
 681       // we're an individual/household/org property
 682
 683       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 684
 685       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 686       return TRUE;
 687     }
 688
 689     if (isset($values['individual_prefix'])) {
 690       if (!empty($params['prefix_id'])) {
 691         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 692         $params['prefix'] = $prefixes[$params['prefix_id']];
 693       }
 694       else {
 695         $params['prefix'] = $values['individual_prefix'];
 696       }
 697       return TRUE;
 698     }
 699
 700     if (isset($values['individual_suffix'])) {
 701       if (!empty($params['suffix_id'])) {
 702         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 703         $params['suffix'] = $suffixes[$params['suffix_id']];
 704       }
 705       else {
 706         $params['suffix'] = $values['individual_suffix'];
 707       }
 708       return TRUE;
 709     }
 710
 711     // CRM-4575
 712     if (isset($values['email_greeting'])) {
 713       if (!empty($params['email_greeting_id'])) {
 714         $emailGreetingFilter = [
 715           'contact_type' => $params['contact_type'] ?? NULL,
 716           'greeting_type' => 'email_greeting',
 717         ];
 718         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
 719         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
 720       }
 721       else {
 722         $params['email_greeting'] = $values['email_greeting'];
 723       }
 724
 725       return TRUE;
 726     }
 727
 728     if (isset($values['postal_greeting'])) {
 729       if (!empty($params['postal_greeting_id'])) {
 730         $postalGreetingFilter = [
 731           'contact_type' => $params['contact_type'] ?? NULL,
 732           'greeting_type' => 'postal_greeting',
 733         ];
 734         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
 735         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
 736       }
 737       else {
 738         $params['postal_greeting'] = $values['postal_greeting'];
 739       }
 740       return TRUE;
 741     }
 742
 743     if (isset($values['addressee'])) {
 744       $params['addressee'] = $values['addressee'];
 745       return TRUE;
 746     }
 747
 748     if (isset($values['gender'])) {
 749       if (!empty($params['gender_id'])) {
 750         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 751         $params['gender'] = $genders[$params['gender_id']];
 752       }
 753       else {
 754         $params['gender'] = $values['gender'];
 755       }
 756       return TRUE;
 757     }
 758
 759     if (!empty($values['preferred_communication_method'])) {
 760       $comm = [];
 761       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
 762
 763       $preffComm = explode(',', $values['preferred_communication_method']);
 764       foreach ($preffComm as $v) {
 765         $v = strtolower(trim($v));
 766         if (array_key_exists($v, $pcm)) {
 767           $comm[$pcm[$v]] = 1;
 768         }
 769       }
 770
 771       $params['preferred_communication_method'] = $comm;
 772       return TRUE;
 773     }
 774
 775     // format the website params.
 776     if (!empty($values['url'])) {
 777       static $websiteFields;
 778       if (!is_array($websiteFields)) {
 779         require_once 'CRM/Core/DAO/Website.php';
 780         $websiteFields = CRM_Core_DAO_Website::fields();
 781       }
 782       if (!array_key_exists('website', $params) ||
 783         !is_array($params['website'])
 784       ) {
 785         $params['website'] = [];
 786       }
 787
 788       $websiteCount = count($params['website']);
 789       _civicrm_api3_store_values($websiteFields, $values,
 790         $params['website'][++$websiteCount]
 791       );
 792
 793       return TRUE;
 794     }
 795
 796     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 797     if (!empty($values['location_type_id'])) {
 798       static $fields = NULL;
 799       if ($fields == NULL) {
 800         $fields = [];
 801       }
 802
 803       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 804         $name = strtolower($block);
 805         if (!array_key_exists($name, $values)) {
 806           continue;
 807         }
 808
 809         if ($name === 'phone_ext') {
 810           $block = 'Phone';
 811         }
 812
 813         // block present in value array.
 814         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 815           $params[$name] = [];
 816         }
 817
 818         if (!array_key_exists($block, $fields)) {
 819           $className = "CRM_Core_DAO_$block";
 820           $fields[$block] =& $className::fields();
 821         }
 822
 823         $blockCnt = count($params[$name]);
 824
 825         // copy value to dao field name.
 826         if ($name == 'im') {
 827           $values['name'] = $values[$name];
 828         }
 829
 830         _civicrm_api3_store_values($fields[$block], $values,
 831           $params[$name][++$blockCnt]
 832         );
 833
 834         if (empty($params['id']) && ($blockCnt == 1)) {
 835           $params[$name][$blockCnt]['is_primary'] = TRUE;
 836         }
 837
 838         // we only process single block at a time.
 839         return TRUE;
 840       }
 841
 842       // handle address fields.
 843       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
 844         $params['address'] = [];
 845       }
 846
 847       $addressCnt = 1;
 848       foreach ($params['address'] as $cnt => $addressBlock) {
 849         if (CRM_Utils_Array::value('location_type_id', $values) ==
 850           CRM_Utils_Array::value('location_type_id', $addressBlock)
 851         ) {
 852           $addressCnt = $cnt;
 853           break;
 854         }
 855         $addressCnt++;
 856       }
 857
 858       if (!array_key_exists('Address', $fields)) {
 859         $fields['Address'] = CRM_Core_DAO_Address::fields();
 860       }
 861
 862       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
 863       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
 864       // the address in CRM_Core_BAO_Address::create method
 865       if (!empty($values['location_type_id'])) {
 866         static $customFields = [];
 867         if (empty($customFields)) {
 868           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
 869         }
 870         // make a copy of values, as we going to make changes
 871         $newValues = $values;
 872         foreach ($values as $key => $val) {
 873           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 874           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
 875             // mark an entry in fields array since we want the value of custom field to be copied
 876             $fields['Address'][$key] = NULL;
 877
 878             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 879             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
 880               $mulValues = explode(',', $val);
 881               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 882               $newValues[$key] = [];
 883               foreach ($mulValues as $v1) {
 884                 foreach ($customOption as $v2) {
 885                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 886                     (strtolower($v2['value']) == strtolower(trim($v1)))
 887                   ) {
 888                     if ($htmlType == 'CheckBox') {
 889                       $newValues[$key][$v2['value']] = 1;
 890                     }
 891                     else {
 892                       $newValues[$key][] = $v2['value'];
 893                     }
 894                   }
 895                 }
 896               }
 897             }
 898           }
 899         }
 900         // consider new values
 901         $values = $newValues;
 902       }
 903
 904       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
 905
 906       $addressFields = [
 907         'county',
 908         'country',
 909         'state_province',
 910         'supplemental_address_1',
 911         'supplemental_address_2',
 912         'supplemental_address_3',
 913         'StateProvince.name',
 914       ];
 915
 916       foreach ($addressFields as $field) {
 917         if (array_key_exists($field, $values)) {
 918           if (!array_key_exists('address', $params)) {
 919             $params['address'] = [];
 920           }
 921           $params['address'][$addressCnt][$field] = $values[$field];
 922         }
 923       }
 924
 925       if ($addressCnt == 1) {
 926
 927         $params['address'][$addressCnt]['is_primary'] = TRUE;
 928       }
 929       return TRUE;
 930     }
 931
 932     if (isset($values['note'])) {
 933       // add a note field
 934       if (!isset($params['note'])) {
 935         $params['note'] = [];
 936       }
 937       $noteBlock = count($params['note']) + 1;
 938
 939       $params['note'][$noteBlock] = [];
 940       if (!isset($fields['Note'])) {
 941         $fields['Note'] = CRM_Core_DAO_Note::fields();
 942       }
 943
 944       // get the current logged in civicrm user
 945       $session = CRM_Core_Session::singleton();
 946       $userID = $session->get('userID');
 947
 948       if ($userID) {
 949         $values['contact_id'] = $userID;
 950       }
 951
 952       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
 953
 954       return TRUE;
 955     }
 956
 957     // Check for custom field values
 958
 959     if (empty($fields['custom'])) {
 960       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
 961         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
 962       );
 963     }
 964
 965     foreach ($values as $key => $value) {
 966       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 967         // check if it's a valid custom field id
 968
 969         if (!array_key_exists($customFieldID, $fields['custom'])) {
 970           return civicrm_api3_create_error('Invalid custom field ID');
 971         }
 972         else {
 973           $params[$key] = $value;
 974         }
 975       }
 976     }
 977   }
 978
 979   /**
 980    * Parse a field which could be represented by a label or name value rather than the DB value.
 981    *
 982    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
 983    *
 984    * but if not available then see if we have a label that can be converted to a name.
 985    *
 986    * @param string|int|null $submittedValue
 987    * @param array $fieldSpec
 988    *   Metadata for the field
 989    *
 990    * @return mixed
 991    */
 992   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
 993     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
 994     if (!isset($fieldSpec['bao'])) {
 995       return $submittedValue;
 996     }
 997     /* @var \CRM_Core_DAO $bao */
 998     $bao = $fieldSpec['bao'];
 999     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1000     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1001     if (isset($nameOptions[$submittedValue])) {
1002       return $submittedValue;
1003     }
1004     if (in_array($submittedValue, $nameOptions)) {
1005       return array_search($submittedValue, $nameOptions, TRUE);
1006     }
1007
1008     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1009     if (isset($labelOptions[$submittedValue])) {
1010       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1011     }
1012     return '';
1013   }
1014
1015   /**
1016    * This is code extracted from 4 places where this exact snippet was being duplicated.
1017    *
1018    * FIXME: Extracting this was a first step, but there's also
1019    *  1. Inconsistency in the way other select options are handled.
1020    *     Contribution adds handling for Select/Radio/Autocomplete
1021    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1022    *     Membership is missing all of it
1023    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1024    *
1025    * @param $customFieldID
1026    * @param $value
1027    * @param $fieldType
1028    * @return array
1029    */
1030   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1031     $mulValues = explode(',', $value);
1032     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1033     $values = [];
1034     foreach ($mulValues as $v1) {
1035       foreach ($customOption as $customValueID => $customLabel) {
1036         $customValue = $customLabel['value'];
1037         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1038           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1039         ) {
1040           $values[] = $customValue;
1041         }
1042       }
1043     }
1044     return $values;
1045   }
1046
1047   /**
1048    * Get the ids of any contacts that match according to the rule.
1049    *
1050    * @param array $formatted
1051    *
1052    * @return array
1053    */
1054   protected function getIdsOfMatchingContacts(array $formatted):array {
1055     // the call to the deprecated function seems to add no value other that to do an additional
1056     // check for the contact_id & type.
1057     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1058     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
1059       return [];
1060     }
1061     if (is_array($error['error_message']['params'][0])) {
1062       return $error['error_message']['params'][0];
1063     }
1064     else {
1065       return explode(',', $error['error_message']['params'][0]);
1066     }
1067   }
1068
1069 }