CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Import_Parser {
  18   /**
  19    * Settings
  20    */
  21   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  22
  23   /**
  24    * Return codes
  25    */
  26   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  27
  28   /**
  29    * Parser modes
  30    */
  31   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  32
  33   /**
  34    * Codes for duplicate record handling
  35    */
  36   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  37
  38   /**
  39    * Contact types
  40    */
  41   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  42
  43
  44   /**
  45    * Total number of non empty lines
  46    * @var int
  47    */
  48   protected $_totalCount;
  49
  50   /**
  51    * Running total number of valid lines
  52    * @var int
  53    */
  54   protected $_validCount;
  55
  56   /**
  57    * Running total number of invalid rows
  58    * @var int
  59    */
  60   protected $_invalidRowCount;
  61
  62   /**
  63    * Maximum number of non-empty/comment lines to process
  64    *
  65    * @var int
  66    */
  67   protected $_maxLinesToProcess;
  68
  69   /**
  70    * Array of error lines, bounded by MAX_ERROR
  71    * @var array
  72    */
  73   protected $_errors;
  74
  75   /**
  76    * Total number of conflict lines
  77    * @var int
  78    */
  79   protected $_conflictCount;
  80
  81   /**
  82    * Array of conflict lines
  83    * @var array
  84    */
  85   protected $_conflicts;
  86
  87   /**
  88    * Total number of duplicate (from database) lines
  89    * @var int
  90    */
  91   protected $_duplicateCount;
  92
  93   /**
  94    * Array of duplicate lines
  95    * @var array
  96    */
  97   protected $_duplicates;
  98
  99   /**
 100    * Running total number of warnings
 101    * @var int
 102    */
 103   protected $_warningCount;
 104
 105   /**
 106    * Maximum number of warnings to store
 107    * @var int
 108    */
 109   protected $_maxWarningCount = self::MAX_WARNINGS;
 110
 111   /**
 112    * Array of warning lines, bounded by MAX_WARNING
 113    * @var array
 114    */
 115   protected $_warnings;
 116
 117   /**
 118    * Array of all the fields that could potentially be part
 119    * of this import process
 120    * @var array
 121    */
 122   protected $_fields;
 123
 124   /**
 125    * Metadata for all available fields, keyed by unique name.
 126    *
 127    * This is intended to supercede $_fields which uses a special sauce format which
 128    * importableFieldsMetadata uses the standard getfields type format.
 129    *
 130    * @var array
 131    */
 132   protected $importableFieldsMetadata = [];
 133
 134   /**
 135    * Get metadata for all importable fields in std getfields style format.
 136    *
 137    * @return array
 138    */
 139   public function getImportableFieldsMetadata(): array {
 140     return $this->importableFieldsMetadata;
 141   }
 142
 143   /**
 144    * Set metadata for all importable fields in std getfields style format.
 145    *
 146    * @param array $importableFieldsMetadata
 147    */
 148   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 149     $this->importableFieldsMetadata = $importableFieldsMetadata;
 150   }
 151
 152   /**
 153    * Array of the fields that are actually part of the import process
 154    * the position in the array also dictates their position in the import
 155    * file
 156    * @var array
 157    */
 158   protected $_activeFields;
 159
 160   /**
 161    * Cache the count of active fields
 162    *
 163    * @var int
 164    */
 165   protected $_activeFieldCount;
 166
 167   /**
 168    * Cache of preview rows
 169    *
 170    * @var array
 171    */
 172   protected $_rows;
 173
 174   /**
 175    * Filename of error data
 176    *
 177    * @var string
 178    */
 179   protected $_errorFileName;
 180
 181   /**
 182    * Filename of conflict data
 183    *
 184    * @var string
 185    */
 186   protected $_conflictFileName;
 187
 188   /**
 189    * Filename of duplicate data
 190    *
 191    * @var string
 192    */
 193   protected $_duplicateFileName;
 194
 195   /**
 196    * Contact type
 197    *
 198    * @var int
 199    */
 200   public $_contactType;
 201   /**
 202    * Contact sub-type
 203    *
 204    * @var int
 205    */
 206   public $_contactSubType;
 207
 208   /**
 209    * Class constructor.
 210    */
 211   public function __construct() {
 212     $this->_maxLinesToProcess = 0;
 213   }
 214
 215   /**
 216    * Abstract function definitions.
 217    */
 218   abstract protected function init();
 219
 220   /**
 221    * @return mixed
 222    */
 223   abstract protected function fini();
 224
 225   /**
 226    * Map field.
 227    *
 228    * @param array $values
 229    *
 230    * @return mixed
 231    */
 232   abstract protected function mapField(&$values);
 233
 234   /**
 235    * Preview.
 236    *
 237    * @param array $values
 238    *
 239    * @return mixed
 240    */
 241   abstract protected function preview(&$values);
 242
 243   /**
 244    * @param $values
 245    *
 246    * @return mixed
 247    */
 248   abstract protected function summary(&$values);
 249
 250   /**
 251    * @param $onDuplicate
 252    * @param $values
 253    *
 254    * @return mixed
 255    */
 256   abstract protected function import($onDuplicate, &$values);
 257
 258   /**
 259    * Set and validate field values.
 260    *
 261    * @param array $elements
 262    *   array.
 263    * @param $erroneousField
 264    *   reference.
 265    *
 266    * @return int
 267    */
 268   public function setActiveFieldValues($elements, &$erroneousField = NULL) {
 269     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 270     for ($i = 0; $i < $maxCount; $i++) {
 271       $this->_activeFields[$i]->setValue($elements[$i]);
 272     }
 273
 274     // reset all the values that we did not have an equivalent import element
 275     for (; $i < $this->_activeFieldCount; $i++) {
 276       $this->_activeFields[$i]->resetValue();
 277     }
 278
 279     // now validate the fields and return false if error
 280     $valid = self::VALID;
 281     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 282       if (!$this->_activeFields[$i]->validate()) {
 283         // no need to do any more validation
 284         $erroneousField = $i;
 285         $valid = self::ERROR;
 286         break;
 287       }
 288     }
 289     return $valid;
 290   }
 291
 292   /**
 293    * Format the field values for input to the api.
 294    *
 295    * @return array
 296    *   (reference) associative array of name/value pairs
 297    */
 298   public function &getActiveFieldParams() {
 299     $params = [];
 300     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 301       if (isset($this->_activeFields[$i]->_value)
 302         && !isset($params[$this->_activeFields[$i]->_name])
 303         && !isset($this->_activeFields[$i]->_related)
 304       ) {
 305
 306         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 307       }
 308     }
 309     return $params;
 310   }
 311
 312   /**
 313    * Add progress bar to the import process. Calculates time remaining, status etc.
 314    *
 315    * @param $statusID
 316    *   status id of the import process saved in $config->uploadDir.
 317    * @param bool $startImport
 318    *   True when progress bar is to be initiated.
 319    * @param $startTimestamp
 320    *   Initial timestamp when the import was started.
 321    * @param $prevTimestamp
 322    *   Previous timestamp when this function was last called.
 323    * @param $totalRowCount
 324    *   Total number of rows in the import file.
 325    *
 326    * @return NULL|$currTimestamp
 327    */
 328   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 329     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 330
 331     if ($startImport) {
 332       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 333       //do not force the browser to display the save dialog, CRM-7640
 334       $contents = json_encode([0, $status]);
 335       file_put_contents($statusFile, $contents);
 336     }
 337     else {
 338       $rowCount = $this->_rowCount ?? $this->_lineCount;
 339       $currTimestamp = time();
 340       $time = ($currTimestamp - $prevTimestamp);
 341       $recordsLeft = $totalRowCount - $rowCount;
 342       if ($recordsLeft < 0) {
 343         $recordsLeft = 0;
 344       }
 345       $estimatedTime = ($recordsLeft / 50) * $time;
 346       $estMinutes = floor($estimatedTime / 60);
 347       $timeFormatted = '';
 348       if ($estMinutes > 1) {
 349         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 350         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 351       }
 352       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 353       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 354       $statusMsg = ts('%1 of %2 records - %3 remaining',
 355         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 356       );
 357       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 358       $contents = json_encode([$processedPercent, $status]);
 359
 360       file_put_contents($statusFile, $contents);
 361       return $currTimestamp;
 362     }
 363   }
 364
 365   /**
 366    * @return array
 367    */
 368   public function getSelectValues(): array {
 369     $values = [];
 370     foreach ($this->_fields as $name => $field) {
 371       $values[$name] = $field->_title;
 372     }
 373     return $values;
 374   }
 375
 376   /**
 377    * @return array
 378    */
 379   public function getSelectTypes() {
 380     $values = [];
 381     foreach ($this->_fields as $name => $field) {
 382       if (isset($field->_hasLocationType)) {
 383         $values[$name] = $field->_hasLocationType;
 384       }
 385     }
 386     return $values;
 387   }
 388
 389   /**
 390    * @return array
 391    */
 392   public function getHeaderPatterns() {
 393     $values = [];
 394     foreach ($this->_fields as $name => $field) {
 395       if (isset($field->_headerPattern)) {
 396         $values[$name] = $field->_headerPattern;
 397       }
 398     }
 399     return $values;
 400   }
 401
 402   /**
 403    * @return array
 404    */
 405   public function getDataPatterns() {
 406     $values = [];
 407     foreach ($this->_fields as $name => $field) {
 408       $values[$name] = $field->_dataPattern;
 409     }
 410     return $values;
 411   }
 412
 413   /**
 414    * Remove single-quote enclosures from a value array (row).
 415    *
 416    * @param array $values
 417    * @param string $enclosure
 418    *
 419    * @return void
 420    */
 421   public static function encloseScrub(&$values, $enclosure = "'") {
 422     if (empty($values)) {
 423       return;
 424     }
 425
 426     foreach ($values as $k => $v) {
 427       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 428     }
 429   }
 430
 431   /**
 432    * Setter function.
 433    *
 434    * @param int $max
 435    *
 436    * @return void
 437    */
 438   public function setMaxLinesToProcess($max) {
 439     $this->_maxLinesToProcess = $max;
 440   }
 441
 442   /**
 443    * Determines the file extension based on error code.
 444    *
 445    * @var int $type error code constant
 446    * @return string
 447    */
 448   public static function errorFileName($type) {
 449     $fileName = NULL;
 450     if (empty($type)) {
 451       return $fileName;
 452     }
 453
 454     $config = CRM_Core_Config::singleton();
 455     $fileName = $config->uploadDir . "sqlImport";
 456     switch ($type) {
 457       case self::ERROR:
 458         $fileName .= '.errors';
 459         break;
 460
 461       case self::CONFLICT:
 462         $fileName .= '.conflicts';
 463         break;
 464
 465       case self::DUPLICATE:
 466         $fileName .= '.duplicates';
 467         break;
 468
 469       case self::NO_MATCH:
 470         $fileName .= '.mismatch';
 471         break;
 472
 473       case self::UNPARSED_ADDRESS_WARNING:
 474         $fileName .= '.unparsedAddress';
 475         break;
 476     }
 477
 478     return $fileName;
 479   }
 480
 481   /**
 482    * Determines the file name based on error code.
 483    *
 484    * @var $type error code constant
 485    * @return string
 486    */
 487   public static function saveFileName($type) {
 488     $fileName = NULL;
 489     if (empty($type)) {
 490       return $fileName;
 491     }
 492     switch ($type) {
 493       case self::ERROR:
 494         $fileName = 'Import_Errors.csv';
 495         break;
 496
 497       case self::CONFLICT:
 498         $fileName = 'Import_Conflicts.csv';
 499         break;
 500
 501       case self::DUPLICATE:
 502         $fileName = 'Import_Duplicates.csv';
 503         break;
 504
 505       case self::NO_MATCH:
 506         $fileName = 'Import_Mismatch.csv';
 507         break;
 508
 509       case self::UNPARSED_ADDRESS_WARNING:
 510         $fileName = 'Import_Unparsed_Address.csv';
 511         break;
 512     }
 513
 514     return $fileName;
 515   }
 516
 517   /**
 518    * Check if contact is a duplicate .
 519    *
 520    * @param array $formatValues
 521    *
 522    * @return array
 523    */
 524   protected function checkContactDuplicate(&$formatValues) {
 525     //retrieve contact id using contact dedupe rule
 526     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 527     $formatValues['version'] = 3;
 528     require_once 'CRM/Utils/DeprecatedUtils.php';
 529     $params = $formatValues;
 530     static $cIndieFields = NULL;
 531     static $defaultLocationId = NULL;
 532
 533     $contactType = $params['contact_type'];
 534     if ($cIndieFields == NULL) {
 535       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 536       $cIndieFields = $cTempIndieFields;
 537
 538       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 539
 540       // set the value to default location id else set to 1
 541       if (!$defaultLocationId = (int) $defaultLocation->id) {
 542         $defaultLocationId = 1;
 543       }
 544     }
 545
 546     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 547
 548     $contactFormatted = [];
 549     foreach ($params as $key => $field) {
 550       if ($field == NULL || $field === '') {
 551         continue;
 552       }
 553       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 554       // instead of soft credit contact.
 555       if (is_array($field) && $key != "soft_credit") {
 556         foreach ($field as $value) {
 557           $break = FALSE;
 558           if (is_array($value)) {
 559             foreach ($value as $name => $testForEmpty) {
 560               if ($name !== 'phone_type' &&
 561                 ($testForEmpty === '' || $testForEmpty == NULL)
 562               ) {
 563                 $break = TRUE;
 564                 break;
 565               }
 566             }
 567           }
 568           else {
 569             $break = TRUE;
 570           }
 571           if (!$break) {
 572             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 573           }
 574         }
 575         continue;
 576       }
 577
 578       $value = [$key => $field];
 579
 580       // check if location related field, then we need to add primary location type
 581       if (in_array($key, $locationFields)) {
 582         $value['location_type_id'] = $defaultLocationId;
 583       }
 584       elseif (array_key_exists($key, $cIndieFields)) {
 585         $value['contact_type'] = $contactType;
 586       }
 587
 588       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 589     }
 590
 591     $contactFormatted['contact_type'] = $contactType;
 592
 593     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 594   }
 595
 596   /**
 597    * This function adds the contact variable in $values to the
 598    * parameter list $params.  For most cases, $values should have length 1.  If
 599    * the variable being added is a child of Location, a location_type_id must
 600    * also be included.  If it is a child of phone, a phone_type must be included.
 601    *
 602    * @param array $values
 603    *   The variable(s) to be added.
 604    * @param array $params
 605    *   The structured parameter list.
 606    *
 607    * @return bool|CRM_Utils_Error
 608    */
 609   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 610     // @todo - like most functions in import ... most of this is cruft....
 611     // Crawl through the possible classes:
 612     // Contact
 613     //      Individual
 614     //      Household
 615     //      Organization
 616     //          Location
 617     //              Address
 618     //              Email
 619     //              Phone
 620     //              IM
 621     //      Note
 622     //      Custom
 623
 624     // Cache the various object fields
 625     static $fields = NULL;
 626
 627     if ($fields == NULL) {
 628       $fields = [];
 629     }
 630
 631     // first add core contact values since for other Civi modules they are not added
 632     require_once 'CRM/Contact/BAO/Contact.php';
 633     $contactFields = CRM_Contact_DAO_Contact::fields();
 634     _civicrm_api3_store_values($contactFields, $values, $params);
 635
 636     if (isset($values['contact_type'])) {
 637       // we're an individual/household/org property
 638
 639       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 640
 641       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 642       return TRUE;
 643     }
 644
 645     if (isset($values['individual_prefix'])) {
 646       if (!empty($params['prefix_id'])) {
 647         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 648         $params['prefix'] = $prefixes[$params['prefix_id']];
 649       }
 650       else {
 651         $params['prefix'] = $values['individual_prefix'];
 652       }
 653       return TRUE;
 654     }
 655
 656     if (isset($values['individual_suffix'])) {
 657       if (!empty($params['suffix_id'])) {
 658         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 659         $params['suffix'] = $suffixes[$params['suffix_id']];
 660       }
 661       else {
 662         $params['suffix'] = $values['individual_suffix'];
 663       }
 664       return TRUE;
 665     }
 666
 667     // CRM-4575
 668     if (isset($values['email_greeting'])) {
 669       if (!empty($params['email_greeting_id'])) {
 670         $emailGreetingFilter = [
 671           'contact_type' => $params['contact_type'] ?? NULL,
 672           'greeting_type' => 'email_greeting',
 673         ];
 674         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
 675         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
 676       }
 677       else {
 678         $params['email_greeting'] = $values['email_greeting'];
 679       }
 680
 681       return TRUE;
 682     }
 683
 684     if (isset($values['postal_greeting'])) {
 685       if (!empty($params['postal_greeting_id'])) {
 686         $postalGreetingFilter = [
 687           'contact_type' => $params['contact_type'] ?? NULL,
 688           'greeting_type' => 'postal_greeting',
 689         ];
 690         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
 691         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
 692       }
 693       else {
 694         $params['postal_greeting'] = $values['postal_greeting'];
 695       }
 696       return TRUE;
 697     }
 698
 699     if (isset($values['addressee'])) {
 700       $params['addressee'] = $values['addressee'];
 701       return TRUE;
 702     }
 703
 704     if (isset($values['gender'])) {
 705       if (!empty($params['gender_id'])) {
 706         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 707         $params['gender'] = $genders[$params['gender_id']];
 708       }
 709       else {
 710         $params['gender'] = $values['gender'];
 711       }
 712       return TRUE;
 713     }
 714
 715     if (!empty($values['preferred_communication_method'])) {
 716       $comm = [];
 717       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
 718
 719       $preffComm = explode(',', $values['preferred_communication_method']);
 720       foreach ($preffComm as $v) {
 721         $v = strtolower(trim($v));
 722         if (array_key_exists($v, $pcm)) {
 723           $comm[$pcm[$v]] = 1;
 724         }
 725       }
 726
 727       $params['preferred_communication_method'] = $comm;
 728       return TRUE;
 729     }
 730
 731     // format the website params.
 732     if (!empty($values['url'])) {
 733       static $websiteFields;
 734       if (!is_array($websiteFields)) {
 735         require_once 'CRM/Core/DAO/Website.php';
 736         $websiteFields = CRM_Core_DAO_Website::fields();
 737       }
 738       if (!array_key_exists('website', $params) ||
 739         !is_array($params['website'])
 740       ) {
 741         $params['website'] = [];
 742       }
 743
 744       $websiteCount = count($params['website']);
 745       _civicrm_api3_store_values($websiteFields, $values,
 746         $params['website'][++$websiteCount]
 747       );
 748
 749       return TRUE;
 750     }
 751
 752     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 753     if (!empty($values['location_type_id'])) {
 754       static $fields = NULL;
 755       if ($fields == NULL) {
 756         $fields = [];
 757       }
 758
 759       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 760         $name = strtolower($block);
 761         if (!array_key_exists($name, $values)) {
 762           continue;
 763         }
 764
 765         if ($name === 'phone_ext') {
 766           $block = 'Phone';
 767         }
 768
 769         // block present in value array.
 770         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 771           $params[$name] = [];
 772         }
 773
 774         if (!array_key_exists($block, $fields)) {
 775           $className = "CRM_Core_DAO_$block";
 776           $fields[$block] =& $className::fields();
 777         }
 778
 779         $blockCnt = count($params[$name]);
 780
 781         // copy value to dao field name.
 782         if ($name == 'im') {
 783           $values['name'] = $values[$name];
 784         }
 785
 786         _civicrm_api3_store_values($fields[$block], $values,
 787           $params[$name][++$blockCnt]
 788         );
 789
 790         if (empty($params['id']) && ($blockCnt == 1)) {
 791           $params[$name][$blockCnt]['is_primary'] = TRUE;
 792         }
 793
 794         // we only process single block at a time.
 795         return TRUE;
 796       }
 797
 798       // handle address fields.
 799       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
 800         $params['address'] = [];
 801       }
 802
 803       $addressCnt = 1;
 804       foreach ($params['address'] as $cnt => $addressBlock) {
 805         if (CRM_Utils_Array::value('location_type_id', $values) ==
 806           CRM_Utils_Array::value('location_type_id', $addressBlock)
 807         ) {
 808           $addressCnt = $cnt;
 809           break;
 810         }
 811         $addressCnt++;
 812       }
 813
 814       if (!array_key_exists('Address', $fields)) {
 815         $fields['Address'] = CRM_Core_DAO_Address::fields();
 816       }
 817
 818       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
 819       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
 820       // the address in CRM_Core_BAO_Address::create method
 821       if (!empty($values['location_type_id'])) {
 822         static $customFields = [];
 823         if (empty($customFields)) {
 824           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
 825         }
 826         // make a copy of values, as we going to make changes
 827         $newValues = $values;
 828         foreach ($values as $key => $val) {
 829           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 830           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
 831             // mark an entry in fields array since we want the value of custom field to be copied
 832             $fields['Address'][$key] = NULL;
 833
 834             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 835             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
 836               $mulValues = explode(',', $val);
 837               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 838               $newValues[$key] = [];
 839               foreach ($mulValues as $v1) {
 840                 foreach ($customOption as $v2) {
 841                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 842                     (strtolower($v2['value']) == strtolower(trim($v1)))
 843                   ) {
 844                     if ($htmlType == 'CheckBox') {
 845                       $newValues[$key][$v2['value']] = 1;
 846                     }
 847                     else {
 848                       $newValues[$key][] = $v2['value'];
 849                     }
 850                   }
 851                 }
 852               }
 853             }
 854           }
 855         }
 856         // consider new values
 857         $values = $newValues;
 858       }
 859
 860       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
 861
 862       $addressFields = [
 863         'county',
 864         'country',
 865         'state_province',
 866         'supplemental_address_1',
 867         'supplemental_address_2',
 868         'supplemental_address_3',
 869         'StateProvince.name',
 870       ];
 871
 872       foreach ($addressFields as $field) {
 873         if (array_key_exists($field, $values)) {
 874           if (!array_key_exists('address', $params)) {
 875             $params['address'] = [];
 876           }
 877           $params['address'][$addressCnt][$field] = $values[$field];
 878         }
 879       }
 880
 881       if ($addressCnt == 1) {
 882
 883         $params['address'][$addressCnt]['is_primary'] = TRUE;
 884       }
 885       return TRUE;
 886     }
 887
 888     if (isset($values['note'])) {
 889       // add a note field
 890       if (!isset($params['note'])) {
 891         $params['note'] = [];
 892       }
 893       $noteBlock = count($params['note']) + 1;
 894
 895       $params['note'][$noteBlock] = [];
 896       if (!isset($fields['Note'])) {
 897         $fields['Note'] = CRM_Core_DAO_Note::fields();
 898       }
 899
 900       // get the current logged in civicrm user
 901       $session = CRM_Core_Session::singleton();
 902       $userID = $session->get('userID');
 903
 904       if ($userID) {
 905         $values['contact_id'] = $userID;
 906       }
 907
 908       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
 909
 910       return TRUE;
 911     }
 912
 913     // Check for custom field values
 914
 915     if (empty($fields['custom'])) {
 916       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
 917         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
 918       );
 919     }
 920
 921     foreach ($values as $key => $value) {
 922       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 923         // check if it's a valid custom field id
 924
 925         if (!array_key_exists($customFieldID, $fields['custom'])) {
 926           return civicrm_api3_create_error('Invalid custom field ID');
 927         }
 928         else {
 929           $params[$key] = $value;
 930         }
 931       }
 932     }
 933   }
 934
 935   /**
 936    * Parse a field which could be represented by a label or name value rather than the DB value.
 937    *
 938    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
 939    *
 940    * but if not available then see if we have a label that can be converted to a name.
 941    *
 942    * @param string|int|null $submittedValue
 943    * @param array $fieldSpec
 944    *   Metadata for the field
 945    *
 946    * @return mixed
 947    */
 948   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
 949     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
 950     if (!isset($fieldSpec['bao'])) {
 951       return $submittedValue;
 952     }
 953     /* @var \CRM_Core_DAO $bao */
 954     $bao = $fieldSpec['bao'];
 955     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
 956     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
 957     if (isset($nameOptions[$submittedValue])) {
 958       return $submittedValue;
 959     }
 960     if (in_array($submittedValue, $nameOptions)) {
 961       return array_search($submittedValue, $nameOptions, TRUE);
 962     }
 963
 964     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
 965     if (isset($labelOptions[$submittedValue])) {
 966       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
 967     }
 968     return '';
 969   }
 970
 971   /**
 972    * This is code extracted from 4 places where this exact snippet was being duplicated.
 973    *
 974    * FIXME: Extracting this was a first step, but there's also
 975    *  1. Inconsistency in the way other select options are handled.
 976    *     Contribution adds handling for Select/Radio/Autocomplete
 977    *     Participant/Activity only handles Select/Radio and misses Autocomplete
 978    *     Membership is missing all of it
 979    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
 980    *
 981    * @param $customFieldID
 982    * @param $value
 983    * @param $fieldType
 984    * @return array
 985    */
 986   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
 987     $mulValues = explode(',', $value);
 988     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 989     $values = [];
 990     foreach ($mulValues as $v1) {
 991       foreach ($customOption as $customValueID => $customLabel) {
 992         $customValue = $customLabel['value'];
 993         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
 994           (strtolower(trim($customValue)) == strtolower(trim($v1)))
 995         ) {
 996           if ($fieldType == 'CheckBox') {
 997             $values[$customValue] = 1;
 998           }
 999           else {
1000             $values[] = $customValue;
1001           }
1002         }
1003       }
1004     }
1005     return $values;
1006   }
1007
1008   /**
1009    * Get the ids of any contacts that match according to the rule.
1010    *
1011    * @param array $formatted
1012    *
1013    * @return array
1014    */
1015   protected function getIdsOfMatchingContacts(array $formatted):array {
1016     // the call to the deprecated function seems to add no value other that to do an additional
1017     // check for the contact_id & type.
1018     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1019     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
1020       return [];
1021     }
1022     if (is_array($error['error_message']['params'][0])) {
1023       return $error['error_message']['params'][0];
1024     }
1025     else {
1026       return explode(',', $error['error_message']['params'][0]);
1027     }
1028   }
1029
1030 }