CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\UserJob;
  13
  14 /**
  15  *
  16  * @package CRM
  17  * @copyright CiviCRM LLC https://civicrm.org/licensing
  18  */
  19 abstract class CRM_Import_Parser {
  20   /**
  21    * Settings
  22    */
  23   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  24
  25   /**
  26    * Return codes
  27    */
  28   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  29
  30   /**
  31    * Parser modes
  32    */
  33   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  34
  35   /**
  36    * Codes for duplicate record handling
  37    */
  38   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  39
  40   /**
  41    * Contact types
  42    */
  43   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  44
  45
  46   /**
  47    * User job id.
  48    *
  49    * This is the primary key of the civicrm_user_job table which is used to
  50    * track the import.
  51    *
  52    * @var int
  53    */
  54   protected $userJobID;
  55
  56   /**
  57    * @return int|null
  58    */
  59   public function getUserJobID(): ?int {
  60     return $this->userJobID;
  61   }
  62
  63   /**
  64    * Set user job ID.
  65    *
  66    * @param int $userJobID
  67    */
  68   public function setUserJobID(int $userJobID): void {
  69     $this->userJobID = $userJobID;
  70   }
  71
  72   /**
  73    * Get User Job.
  74    *
  75    * API call to retrieve the userJob row.
  76    *
  77    * @return array
  78    *
  79    * @throws \API_Exception
  80    */
  81   protected function getUserJob(): array {
  82     return UserJob::get()
  83       ->addWhere('id', '=', $this->getUserJobID())
  84       ->execute()
  85       ->first();
  86   }
  87
  88   /**
  89    * Get the submitted value, as stored on the user job.
  90    *
  91    * @param string $fieldName
  92    *
  93    * @return mixed
  94    *
  95    * @throws \API_Exception
  96    */
  97   protected function getSubmittedValue(string $fieldName) {
  98     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
  99   }
 100
 101   /**
 102    * Get configured contact type.
 103    *
 104    * @throws \API_Exception
 105    */
 106   protected function getContactType() {
 107     if (!$this->_contactType) {
 108       $contactTypeMapping = [
 109         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 110         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 111         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 112       ];
 113       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 114     }
 115     return $this->_contactType;
 116   }
 117
 118   /**
 119    * Get configured contact type.
 120    *
 121    * @return string|null
 122    *
 123    * @throws \API_Exception
 124    */
 125   public function getContactSubType() {
 126     if (!$this->_contactSubType) {
 127       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 128     }
 129     return $this->_contactSubType;
 130   }
 131
 132   /**
 133    * Total number of non empty lines
 134    * @var int
 135    */
 136   protected $_totalCount;
 137
 138   /**
 139    * Running total number of valid lines
 140    * @var int
 141    */
 142   protected $_validCount;
 143
 144   /**
 145    * Running total number of invalid rows
 146    * @var int
 147    */
 148   protected $_invalidRowCount;
 149
 150   /**
 151    * Maximum number of non-empty/comment lines to process
 152    *
 153    * @var int
 154    */
 155   protected $_maxLinesToProcess;
 156
 157   /**
 158    * Array of error lines, bounded by MAX_ERROR
 159    * @var array
 160    */
 161   protected $_errors;
 162
 163   /**
 164    * Total number of conflict lines
 165    * @var int
 166    */
 167   protected $_conflictCount;
 168
 169   /**
 170    * Array of conflict lines
 171    * @var array
 172    */
 173   protected $_conflicts;
 174
 175   /**
 176    * Total number of duplicate (from database) lines
 177    * @var int
 178    */
 179   protected $_duplicateCount;
 180
 181   /**
 182    * Array of duplicate lines
 183    * @var array
 184    */
 185   protected $_duplicates;
 186
 187   /**
 188    * Maximum number of warnings to store
 189    * @var int
 190    */
 191   protected $_maxWarningCount = self::MAX_WARNINGS;
 192
 193   /**
 194    * Array of warning lines, bounded by MAX_WARNING
 195    * @var array
 196    */
 197   protected $_warnings;
 198
 199   /**
 200    * Array of all the fields that could potentially be part
 201    * of this import process
 202    * @var array
 203    */
 204   protected $_fields;
 205
 206   /**
 207    * Metadata for all available fields, keyed by unique name.
 208    *
 209    * This is intended to supercede $_fields which uses a special sauce format which
 210    * importableFieldsMetadata uses the standard getfields type format.
 211    *
 212    * @var array
 213    */
 214   protected $importableFieldsMetadata = [];
 215
 216   /**
 217    * Get metadata for all importable fields in std getfields style format.
 218    *
 219    * @return array
 220    */
 221   public function getImportableFieldsMetadata(): array {
 222     return $this->importableFieldsMetadata;
 223   }
 224
 225   /**
 226    * Set metadata for all importable fields in std getfields style format.
 227    *
 228    * @param array $importableFieldsMetadata
 229    */
 230   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 231     $this->importableFieldsMetadata = $importableFieldsMetadata;
 232   }
 233
 234   /**
 235    * Array of the fields that are actually part of the import process
 236    * the position in the array also dictates their position in the import
 237    * file
 238    * @var array
 239    */
 240   protected $_activeFields;
 241
 242   /**
 243    * Cache the count of active fields
 244    *
 245    * @var int
 246    */
 247   protected $_activeFieldCount;
 248
 249   /**
 250    * Cache of preview rows
 251    *
 252    * @var array
 253    */
 254   protected $_rows;
 255
 256   /**
 257    * Filename of error data
 258    *
 259    * @var string
 260    */
 261   protected $_errorFileName;
 262
 263   /**
 264    * Filename of conflict data
 265    *
 266    * @var string
 267    */
 268   protected $_conflictFileName;
 269
 270   /**
 271    * Filename of duplicate data
 272    *
 273    * @var string
 274    */
 275   protected $_duplicateFileName;
 276
 277   /**
 278    * Contact type
 279    *
 280    * @var string
 281    */
 282   public $_contactType;
 283
 284   /**
 285    * Contact sub-type
 286    *
 287    * @var int|null
 288    */
 289   public $_contactSubType;
 290
 291   /**
 292    * @param int|null $contactSubType
 293    *
 294    * @return self
 295    */
 296   public function setContactSubType(?int $contactSubType): self {
 297     $this->_contactSubType = $contactSubType;
 298     return $this;
 299   }
 300
 301   /**
 302    * Class constructor.
 303    */
 304   public function __construct() {
 305     $this->_maxLinesToProcess = 0;
 306   }
 307
 308   /**
 309    * Set and validate field values.
 310    *
 311    * @param array $elements
 312    *   array.
 313    */
 314   public function setActiveFieldValues($elements): void {
 315     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 316     for ($i = 0; $i < $maxCount; $i++) {
 317       $this->_activeFields[$i]->setValue($elements[$i]);
 318     }
 319
 320     // reset all the values that we did not have an equivalent import element
 321     for (; $i < $this->_activeFieldCount; $i++) {
 322       $this->_activeFields[$i]->resetValue();
 323     }
 324   }
 325
 326   /**
 327    * Format the field values for input to the api.
 328    *
 329    * @return array
 330    *   (reference) associative array of name/value pairs
 331    */
 332   public function &getActiveFieldParams() {
 333     $params = [];
 334     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 335       if (isset($this->_activeFields[$i]->_value)
 336         && !isset($params[$this->_activeFields[$i]->_name])
 337         && !isset($this->_activeFields[$i]->_related)
 338       ) {
 339
 340         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 341       }
 342     }
 343     return $params;
 344   }
 345
 346   /**
 347    * Add progress bar to the import process. Calculates time remaining, status etc.
 348    *
 349    * @param $statusID
 350    *   status id of the import process saved in $config->uploadDir.
 351    * @param bool $startImport
 352    *   True when progress bar is to be initiated.
 353    * @param $startTimestamp
 354    *   Initial timestamp when the import was started.
 355    * @param $prevTimestamp
 356    *   Previous timestamp when this function was last called.
 357    * @param $totalRowCount
 358    *   Total number of rows in the import file.
 359    *
 360    * @return NULL|$currTimestamp
 361    */
 362   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 363     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 364
 365     if ($startImport) {
 366       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 367       //do not force the browser to display the save dialog, CRM-7640
 368       $contents = json_encode([0, $status]);
 369       file_put_contents($statusFile, $contents);
 370     }
 371     else {
 372       $rowCount = $this->_rowCount ?? $this->_lineCount;
 373       $currTimestamp = time();
 374       $time = ($currTimestamp - $prevTimestamp);
 375       $recordsLeft = $totalRowCount - $rowCount;
 376       if ($recordsLeft < 0) {
 377         $recordsLeft = 0;
 378       }
 379       $estimatedTime = ($recordsLeft / 50) * $time;
 380       $estMinutes = floor($estimatedTime / 60);
 381       $timeFormatted = '';
 382       if ($estMinutes > 1) {
 383         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 384         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 385       }
 386       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 387       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 388       $statusMsg = ts('%1 of %2 records - %3 remaining',
 389         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 390       );
 391       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 392       $contents = json_encode([$processedPercent, $status]);
 393
 394       file_put_contents($statusFile, $contents);
 395       return $currTimestamp;
 396     }
 397   }
 398
 399   /**
 400    * @return array
 401    */
 402   public function getSelectValues(): array {
 403     $values = [];
 404     foreach ($this->_fields as $name => $field) {
 405       $values[$name] = $field->_title;
 406     }
 407     return $values;
 408   }
 409
 410   /**
 411    * @return array
 412    */
 413   public function getSelectTypes() {
 414     $values = [];
 415     foreach ($this->_fields as $name => $field) {
 416       if (isset($field->_hasLocationType)) {
 417         $values[$name] = $field->_hasLocationType;
 418       }
 419     }
 420     return $values;
 421   }
 422
 423   /**
 424    * @return array
 425    */
 426   public function getHeaderPatterns() {
 427     $values = [];
 428     foreach ($this->_fields as $name => $field) {
 429       if (isset($field->_headerPattern)) {
 430         $values[$name] = $field->_headerPattern;
 431       }
 432     }
 433     return $values;
 434   }
 435
 436   /**
 437    * @return array
 438    */
 439   public function getDataPatterns() {
 440     $values = [];
 441     foreach ($this->_fields as $name => $field) {
 442       $values[$name] = $field->_dataPattern;
 443     }
 444     return $values;
 445   }
 446
 447   /**
 448    * Remove single-quote enclosures from a value array (row).
 449    *
 450    * @param array $values
 451    * @param string $enclosure
 452    *
 453    * @return void
 454    */
 455   public static function encloseScrub(&$values, $enclosure = "'") {
 456     if (empty($values)) {
 457       return;
 458     }
 459
 460     foreach ($values as $k => $v) {
 461       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 462     }
 463   }
 464
 465   /**
 466    * Setter function.
 467    *
 468    * @param int $max
 469    *
 470    * @return void
 471    */
 472   public function setMaxLinesToProcess($max) {
 473     $this->_maxLinesToProcess = $max;
 474   }
 475
 476   /**
 477    * Determines the file extension based on error code.
 478    *
 479    * @var int $type error code constant
 480    * @return string
 481    */
 482   public static function errorFileName($type) {
 483     $fileName = NULL;
 484     if (empty($type)) {
 485       return $fileName;
 486     }
 487
 488     $config = CRM_Core_Config::singleton();
 489     $fileName = $config->uploadDir . "sqlImport";
 490     switch ($type) {
 491       case self::ERROR:
 492         $fileName .= '.errors';
 493         break;
 494
 495       case self::CONFLICT:
 496         $fileName .= '.conflicts';
 497         break;
 498
 499       case self::DUPLICATE:
 500         $fileName .= '.duplicates';
 501         break;
 502
 503       case self::NO_MATCH:
 504         $fileName .= '.mismatch';
 505         break;
 506
 507       case self::UNPARSED_ADDRESS_WARNING:
 508         $fileName .= '.unparsedAddress';
 509         break;
 510     }
 511
 512     return $fileName;
 513   }
 514
 515   /**
 516    * Determines the file name based on error code.
 517    *
 518    * @var $type error code constant
 519    * @return string
 520    */
 521   public static function saveFileName($type) {
 522     $fileName = NULL;
 523     if (empty($type)) {
 524       return $fileName;
 525     }
 526     switch ($type) {
 527       case self::ERROR:
 528         $fileName = 'Import_Errors.csv';
 529         break;
 530
 531       case self::CONFLICT:
 532         $fileName = 'Import_Conflicts.csv';
 533         break;
 534
 535       case self::DUPLICATE:
 536         $fileName = 'Import_Duplicates.csv';
 537         break;
 538
 539       case self::NO_MATCH:
 540         $fileName = 'Import_Mismatch.csv';
 541         break;
 542
 543       case self::UNPARSED_ADDRESS_WARNING:
 544         $fileName = 'Import_Unparsed_Address.csv';
 545         break;
 546     }
 547
 548     return $fileName;
 549   }
 550
 551   /**
 552    * Check if contact is a duplicate .
 553    *
 554    * @param array $formatValues
 555    *
 556    * @return array
 557    */
 558   protected function checkContactDuplicate(&$formatValues) {
 559     //retrieve contact id using contact dedupe rule
 560     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 561     $formatValues['version'] = 3;
 562     require_once 'CRM/Utils/DeprecatedUtils.php';
 563     $params = $formatValues;
 564     static $cIndieFields = NULL;
 565     static $defaultLocationId = NULL;
 566
 567     $contactType = $params['contact_type'];
 568     if ($cIndieFields == NULL) {
 569       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 570       $cIndieFields = $cTempIndieFields;
 571
 572       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 573
 574       // set the value to default location id else set to 1
 575       if (!$defaultLocationId = (int) $defaultLocation->id) {
 576         $defaultLocationId = 1;
 577       }
 578     }
 579
 580     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 581
 582     $contactFormatted = [];
 583     foreach ($params as $key => $field) {
 584       if ($field == NULL || $field === '') {
 585         continue;
 586       }
 587       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 588       // instead of soft credit contact.
 589       if (is_array($field) && $key != "soft_credit") {
 590         foreach ($field as $value) {
 591           $break = FALSE;
 592           if (is_array($value)) {
 593             foreach ($value as $name => $testForEmpty) {
 594               if ($name !== 'phone_type' &&
 595                 ($testForEmpty === '' || $testForEmpty == NULL)
 596               ) {
 597                 $break = TRUE;
 598                 break;
 599               }
 600             }
 601           }
 602           else {
 603             $break = TRUE;
 604           }
 605           if (!$break) {
 606             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 607           }
 608         }
 609         continue;
 610       }
 611
 612       $value = [$key => $field];
 613
 614       // check if location related field, then we need to add primary location type
 615       if (in_array($key, $locationFields)) {
 616         $value['location_type_id'] = $defaultLocationId;
 617       }
 618       elseif (array_key_exists($key, $cIndieFields)) {
 619         $value['contact_type'] = $contactType;
 620       }
 621
 622       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 623     }
 624
 625     $contactFormatted['contact_type'] = $contactType;
 626
 627     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 628   }
 629
 630   /**
 631    * This function adds the contact variable in $values to the
 632    * parameter list $params.  For most cases, $values should have length 1.  If
 633    * the variable being added is a child of Location, a location_type_id must
 634    * also be included.  If it is a child of phone, a phone_type must be included.
 635    *
 636    * @param array $values
 637    *   The variable(s) to be added.
 638    * @param array $params
 639    *   The structured parameter list.
 640    *
 641    * @return bool|CRM_Utils_Error
 642    */
 643   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 644     // @todo - like most functions in import ... most of this is cruft....
 645     // Crawl through the possible classes:
 646     // Contact
 647     //      Individual
 648     //      Household
 649     //      Organization
 650     //          Location
 651     //              Address
 652     //              Email
 653     //              Phone
 654     //              IM
 655     //      Note
 656     //      Custom
 657
 658     // Cache the various object fields
 659     static $fields = NULL;
 660
 661     if ($fields == NULL) {
 662       $fields = [];
 663     }
 664
 665     // first add core contact values since for other Civi modules they are not added
 666     require_once 'CRM/Contact/BAO/Contact.php';
 667     $contactFields = CRM_Contact_DAO_Contact::fields();
 668     _civicrm_api3_store_values($contactFields, $values, $params);
 669
 670     if (isset($values['contact_type'])) {
 671       // we're an individual/household/org property
 672
 673       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 674
 675       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 676       return TRUE;
 677     }
 678
 679     if (isset($values['individual_prefix'])) {
 680       if (!empty($params['prefix_id'])) {
 681         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 682         $params['prefix'] = $prefixes[$params['prefix_id']];
 683       }
 684       else {
 685         $params['prefix'] = $values['individual_prefix'];
 686       }
 687       return TRUE;
 688     }
 689
 690     if (isset($values['individual_suffix'])) {
 691       if (!empty($params['suffix_id'])) {
 692         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 693         $params['suffix'] = $suffixes[$params['suffix_id']];
 694       }
 695       else {
 696         $params['suffix'] = $values['individual_suffix'];
 697       }
 698       return TRUE;
 699     }
 700
 701     // CRM-4575
 702     if (isset($values['email_greeting'])) {
 703       if (!empty($params['email_greeting_id'])) {
 704         $emailGreetingFilter = [
 705           'contact_type' => $params['contact_type'] ?? NULL,
 706           'greeting_type' => 'email_greeting',
 707         ];
 708         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
 709         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
 710       }
 711       else {
 712         $params['email_greeting'] = $values['email_greeting'];
 713       }
 714
 715       return TRUE;
 716     }
 717
 718     if (isset($values['postal_greeting'])) {
 719       if (!empty($params['postal_greeting_id'])) {
 720         $postalGreetingFilter = [
 721           'contact_type' => $params['contact_type'] ?? NULL,
 722           'greeting_type' => 'postal_greeting',
 723         ];
 724         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
 725         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
 726       }
 727       else {
 728         $params['postal_greeting'] = $values['postal_greeting'];
 729       }
 730       return TRUE;
 731     }
 732
 733     if (isset($values['addressee'])) {
 734       $params['addressee'] = $values['addressee'];
 735       return TRUE;
 736     }
 737
 738     if (isset($values['gender'])) {
 739       if (!empty($params['gender_id'])) {
 740         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 741         $params['gender'] = $genders[$params['gender_id']];
 742       }
 743       else {
 744         $params['gender'] = $values['gender'];
 745       }
 746       return TRUE;
 747     }
 748
 749     if (!empty($values['preferred_communication_method'])) {
 750       $comm = [];
 751       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
 752
 753       $preffComm = explode(',', $values['preferred_communication_method']);
 754       foreach ($preffComm as $v) {
 755         $v = strtolower(trim($v));
 756         if (array_key_exists($v, $pcm)) {
 757           $comm[$pcm[$v]] = 1;
 758         }
 759       }
 760
 761       $params['preferred_communication_method'] = $comm;
 762       return TRUE;
 763     }
 764
 765     // format the website params.
 766     if (!empty($values['url'])) {
 767       static $websiteFields;
 768       if (!is_array($websiteFields)) {
 769         require_once 'CRM/Core/DAO/Website.php';
 770         $websiteFields = CRM_Core_DAO_Website::fields();
 771       }
 772       if (!array_key_exists('website', $params) ||
 773         !is_array($params['website'])
 774       ) {
 775         $params['website'] = [];
 776       }
 777
 778       $websiteCount = count($params['website']);
 779       _civicrm_api3_store_values($websiteFields, $values,
 780         $params['website'][++$websiteCount]
 781       );
 782
 783       return TRUE;
 784     }
 785
 786     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 787     if (!empty($values['location_type_id'])) {
 788       static $fields = NULL;
 789       if ($fields == NULL) {
 790         $fields = [];
 791       }
 792
 793       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 794         $name = strtolower($block);
 795         if (!array_key_exists($name, $values)) {
 796           continue;
 797         }
 798
 799         if ($name === 'phone_ext') {
 800           $block = 'Phone';
 801         }
 802
 803         // block present in value array.
 804         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 805           $params[$name] = [];
 806         }
 807
 808         if (!array_key_exists($block, $fields)) {
 809           $className = "CRM_Core_DAO_$block";
 810           $fields[$block] =& $className::fields();
 811         }
 812
 813         $blockCnt = count($params[$name]);
 814
 815         // copy value to dao field name.
 816         if ($name == 'im') {
 817           $values['name'] = $values[$name];
 818         }
 819
 820         _civicrm_api3_store_values($fields[$block], $values,
 821           $params[$name][++$blockCnt]
 822         );
 823
 824         if (empty($params['id']) && ($blockCnt == 1)) {
 825           $params[$name][$blockCnt]['is_primary'] = TRUE;
 826         }
 827
 828         // we only process single block at a time.
 829         return TRUE;
 830       }
 831
 832       // handle address fields.
 833       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
 834         $params['address'] = [];
 835       }
 836
 837       $addressCnt = 1;
 838       foreach ($params['address'] as $cnt => $addressBlock) {
 839         if (CRM_Utils_Array::value('location_type_id', $values) ==
 840           CRM_Utils_Array::value('location_type_id', $addressBlock)
 841         ) {
 842           $addressCnt = $cnt;
 843           break;
 844         }
 845         $addressCnt++;
 846       }
 847
 848       if (!array_key_exists('Address', $fields)) {
 849         $fields['Address'] = CRM_Core_DAO_Address::fields();
 850       }
 851
 852       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
 853       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
 854       // the address in CRM_Core_BAO_Address::create method
 855       if (!empty($values['location_type_id'])) {
 856         static $customFields = [];
 857         if (empty($customFields)) {
 858           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
 859         }
 860         // make a copy of values, as we going to make changes
 861         $newValues = $values;
 862         foreach ($values as $key => $val) {
 863           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 864           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
 865             // mark an entry in fields array since we want the value of custom field to be copied
 866             $fields['Address'][$key] = NULL;
 867
 868             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 869             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
 870               $mulValues = explode(',', $val);
 871               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 872               $newValues[$key] = [];
 873               foreach ($mulValues as $v1) {
 874                 foreach ($customOption as $v2) {
 875                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 876                     (strtolower($v2['value']) == strtolower(trim($v1)))
 877                   ) {
 878                     if ($htmlType == 'CheckBox') {
 879                       $newValues[$key][$v2['value']] = 1;
 880                     }
 881                     else {
 882                       $newValues[$key][] = $v2['value'];
 883                     }
 884                   }
 885                 }
 886               }
 887             }
 888           }
 889         }
 890         // consider new values
 891         $values = $newValues;
 892       }
 893
 894       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
 895
 896       $addressFields = [
 897         'county',
 898         'country',
 899         'state_province',
 900         'supplemental_address_1',
 901         'supplemental_address_2',
 902         'supplemental_address_3',
 903         'StateProvince.name',
 904       ];
 905
 906       foreach ($addressFields as $field) {
 907         if (array_key_exists($field, $values)) {
 908           if (!array_key_exists('address', $params)) {
 909             $params['address'] = [];
 910           }
 911           $params['address'][$addressCnt][$field] = $values[$field];
 912         }
 913       }
 914
 915       if ($addressCnt == 1) {
 916
 917         $params['address'][$addressCnt]['is_primary'] = TRUE;
 918       }
 919       return TRUE;
 920     }
 921
 922     if (isset($values['note'])) {
 923       // add a note field
 924       if (!isset($params['note'])) {
 925         $params['note'] = [];
 926       }
 927       $noteBlock = count($params['note']) + 1;
 928
 929       $params['note'][$noteBlock] = [];
 930       if (!isset($fields['Note'])) {
 931         $fields['Note'] = CRM_Core_DAO_Note::fields();
 932       }
 933
 934       // get the current logged in civicrm user
 935       $session = CRM_Core_Session::singleton();
 936       $userID = $session->get('userID');
 937
 938       if ($userID) {
 939         $values['contact_id'] = $userID;
 940       }
 941
 942       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
 943
 944       return TRUE;
 945     }
 946
 947     // Check for custom field values
 948
 949     if (empty($fields['custom'])) {
 950       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
 951         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
 952       );
 953     }
 954
 955     foreach ($values as $key => $value) {
 956       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 957         // check if it's a valid custom field id
 958
 959         if (!array_key_exists($customFieldID, $fields['custom'])) {
 960           return civicrm_api3_create_error('Invalid custom field ID');
 961         }
 962         else {
 963           $params[$key] = $value;
 964         }
 965       }
 966     }
 967   }
 968
 969   /**
 970    * Parse a field which could be represented by a label or name value rather than the DB value.
 971    *
 972    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
 973    *
 974    * but if not available then see if we have a label that can be converted to a name.
 975    *
 976    * @param string|int|null $submittedValue
 977    * @param array $fieldSpec
 978    *   Metadata for the field
 979    *
 980    * @return mixed
 981    */
 982   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
 983     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
 984     if (!isset($fieldSpec['bao'])) {
 985       return $submittedValue;
 986     }
 987     /* @var \CRM_Core_DAO $bao */
 988     $bao = $fieldSpec['bao'];
 989     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
 990     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
 991     if (isset($nameOptions[$submittedValue])) {
 992       return $submittedValue;
 993     }
 994     if (in_array($submittedValue, $nameOptions)) {
 995       return array_search($submittedValue, $nameOptions, TRUE);
 996     }
 997
 998     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
 999     if (isset($labelOptions[$submittedValue])) {
1000       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1001     }
1002     return '';
1003   }
1004
1005   /**
1006    * This is code extracted from 4 places where this exact snippet was being duplicated.
1007    *
1008    * FIXME: Extracting this was a first step, but there's also
1009    *  1. Inconsistency in the way other select options are handled.
1010    *     Contribution adds handling for Select/Radio/Autocomplete
1011    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1012    *     Membership is missing all of it
1013    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1014    *
1015    * @param $customFieldID
1016    * @param $value
1017    * @param $fieldType
1018    * @return array
1019    */
1020   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1021     $mulValues = explode(',', $value);
1022     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1023     $values = [];
1024     foreach ($mulValues as $v1) {
1025       foreach ($customOption as $customValueID => $customLabel) {
1026         $customValue = $customLabel['value'];
1027         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1028           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1029         ) {
1030           $values[] = $customValue;
1031         }
1032       }
1033     }
1034     return $values;
1035   }
1036
1037   /**
1038    * Get the ids of any contacts that match according to the rule.
1039    *
1040    * @param array $formatted
1041    *
1042    * @return array
1043    */
1044   protected function getIdsOfMatchingContacts(array $formatted):array {
1045     // the call to the deprecated function seems to add no value other that to do an additional
1046     // check for the contact_id & type.
1047     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1048     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
1049       return [];
1050     }
1051     if (is_array($error['error_message']['params'][0])) {
1052       return $error['error_message']['params'][0];
1053     }
1054     else {
1055       return explode(',', $error['error_message']['params'][0]);
1056     }
1057   }
1058
1059 }