CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\UserJob;
  13
  14 /**
  15  *
  16  * @package CRM
  17  * @copyright CiviCRM LLC https://civicrm.org/licensing
  18  */
  19 abstract class CRM_Import_Parser {
  20   /**
  21    * Settings
  22    */
  23   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  24
  25   /**
  26    * Return codes
  27    */
  28   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  29
  30   /**
  31    * Parser modes
  32    */
  33   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  34
  35   /**
  36    * Codes for duplicate record handling
  37    */
  38   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  39
  40   /**
  41    * Contact types
  42    */
  43   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  44
  45
  46   /**
  47    * User job id.
  48    *
  49    * This is the primary key of the civicrm_user_job table which is used to
  50    * track the import.
  51    *
  52    * @var int
  53    */
  54   protected $userJobID;
  55
  56   /**
  57    * @return int|null
  58    */
  59   public function getUserJobID(): ?int {
  60     return $this->userJobID;
  61   }
  62
  63   /**
  64    * Set user job ID.
  65    *
  66    * @param int $userJobID
  67    *
  68    * @return self
  69    */
  70   public function setUserJobID(int $userJobID): self {
  71     $this->userJobID = $userJobID;
  72     return $this;
  73   }
  74
  75   /**
  76    * Get User Job.
  77    *
  78    * API call to retrieve the userJob row.
  79    *
  80    * @return array
  81    *
  82    * @throws \API_Exception
  83    */
  84   protected function getUserJob(): array {
  85     return UserJob::get()
  86       ->addWhere('id', '=', $this->getUserJobID())
  87       ->execute()
  88       ->first();
  89   }
  90
  91   /**
  92    * Get the relevant datasource object.
  93    *
  94    * @return \CRM_Import_DataSource|null
  95    *
  96    * @throws \API_Exception
  97    */
  98   protected function getDataSourceObject(): ?CRM_Import_DataSource {
  99     $className = $this->getSubmittedValue('dataSource');
 100     if ($className) {
 101       /* @var CRM_Import_DataSource $dataSource */
 102       return new $className($this->getUserJobID());
 103     }
 104     return NULL;
 105   }
 106
 107   /**
 108    * Get the submitted value, as stored on the user job.
 109    *
 110    * @param string $fieldName
 111    *
 112    * @return mixed
 113    *
 114    * @throws \API_Exception
 115    */
 116   protected function getSubmittedValue(string $fieldName) {
 117     return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
 118   }
 119
 120   /**
 121    * Has the import completed.
 122    *
 123    * @return bool
 124    *
 125    * @throws \API_Exception
 126    * @throws \CRM_Core_Exception
 127    */
 128   public function isComplete() :bool {
 129     return $this->getDataSourceObject()->isCompleted();
 130   }
 131
 132   /**
 133    * Get configured contact type.
 134    *
 135    * @throws \API_Exception
 136    */
 137   protected function getContactType() {
 138     if (!$this->_contactType) {
 139       $contactTypeMapping = [
 140         CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 141         CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 142         CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 143       ];
 144       $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')];
 145     }
 146     return $this->_contactType;
 147   }
 148
 149   /**
 150    * Get configured contact type.
 151    *
 152    * @return string|null
 153    *
 154    * @throws \API_Exception
 155    */
 156   public function getContactSubType() {
 157     if (!$this->_contactSubType) {
 158       $this->_contactSubType = $this->getSubmittedValue('contactSubType');
 159     }
 160     return $this->_contactSubType;
 161   }
 162
 163   /**
 164    * Total number of non empty lines
 165    * @var int
 166    */
 167   protected $_totalCount;
 168
 169   /**
 170    * Running total number of valid lines
 171    * @var int
 172    */
 173   protected $_validCount;
 174
 175   /**
 176    * Running total number of invalid rows
 177    * @var int
 178    */
 179   protected $_invalidRowCount;
 180
 181   /**
 182    * Maximum number of non-empty/comment lines to process
 183    *
 184    * @var int
 185    */
 186   protected $_maxLinesToProcess;
 187
 188   /**
 189    * Array of error lines, bounded by MAX_ERROR
 190    * @var array
 191    */
 192   protected $_errors;
 193
 194   /**
 195    * Total number of duplicate (from database) lines
 196    * @var int
 197    */
 198   protected $_duplicateCount;
 199
 200   /**
 201    * Array of duplicate lines
 202    * @var array
 203    */
 204   protected $_duplicates;
 205
 206   /**
 207    * Maximum number of warnings to store
 208    * @var int
 209    */
 210   protected $_maxWarningCount = self::MAX_WARNINGS;
 211
 212   /**
 213    * Array of warning lines, bounded by MAX_WARNING
 214    * @var array
 215    */
 216   protected $_warnings;
 217
 218   /**
 219    * Array of all the fields that could potentially be part
 220    * of this import process
 221    * @var array
 222    */
 223   protected $_fields;
 224
 225   /**
 226    * Metadata for all available fields, keyed by unique name.
 227    *
 228    * This is intended to supercede $_fields which uses a special sauce format which
 229    * importableFieldsMetadata uses the standard getfields type format.
 230    *
 231    * @var array
 232    */
 233   protected $importableFieldsMetadata = [];
 234
 235   /**
 236    * Get metadata for all importable fields in std getfields style format.
 237    *
 238    * @return array
 239    */
 240   public function getImportableFieldsMetadata(): array {
 241     return $this->importableFieldsMetadata;
 242   }
 243
 244   /**
 245    * Set metadata for all importable fields in std getfields style format.
 246    *
 247    * @param array $importableFieldsMetadata
 248    */
 249   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 250     $this->importableFieldsMetadata = $importableFieldsMetadata;
 251   }
 252
 253   /**
 254    * Array of the fields that are actually part of the import process
 255    * the position in the array also dictates their position in the import
 256    * file
 257    * @var array
 258    */
 259   protected $_activeFields;
 260
 261   /**
 262    * Cache the count of active fields
 263    *
 264    * @var int
 265    */
 266   protected $_activeFieldCount;
 267
 268   /**
 269    * Cache of preview rows
 270    *
 271    * @var array
 272    */
 273   protected $_rows;
 274
 275   /**
 276    * Filename of error data
 277    *
 278    * @var string
 279    */
 280   protected $_errorFileName;
 281
 282   /**
 283    * Filename of duplicate data
 284    *
 285    * @var string
 286    */
 287   protected $_duplicateFileName;
 288
 289   /**
 290    * Contact type
 291    *
 292    * @var string
 293    */
 294   public $_contactType;
 295
 296   /**
 297    * @param string $contactType
 298    *
 299    * @return CRM_Import_Parser
 300    */
 301   public function setContactType(string $contactType): CRM_Import_Parser {
 302     $this->_contactType = $contactType;
 303     return $this;
 304   }
 305
 306   /**
 307    * Contact sub-type
 308    *
 309    * @var int|null
 310    */
 311   public $_contactSubType;
 312
 313   /**
 314    * @param int|null $contactSubType
 315    *
 316    * @return self
 317    */
 318   public function setContactSubType(?int $contactSubType): self {
 319     $this->_contactSubType = $contactSubType;
 320     return $this;
 321   }
 322
 323   /**
 324    * Class constructor.
 325    */
 326   public function __construct() {
 327     $this->_maxLinesToProcess = 0;
 328   }
 329
 330   /**
 331    * Set and validate field values.
 332    *
 333    * @param array $elements
 334    *   array.
 335    */
 336   public function setActiveFieldValues($elements): void {
 337     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 338     for ($i = 0; $i < $maxCount; $i++) {
 339       $this->_activeFields[$i]->setValue($elements[$i]);
 340     }
 341
 342     // reset all the values that we did not have an equivalent import element
 343     for (; $i < $this->_activeFieldCount; $i++) {
 344       $this->_activeFields[$i]->resetValue();
 345     }
 346   }
 347
 348   /**
 349    * Format the field values for input to the api.
 350    *
 351    * @return array
 352    *   (reference) associative array of name/value pairs
 353    */
 354   public function &getActiveFieldParams() {
 355     $params = [];
 356     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 357       if (isset($this->_activeFields[$i]->_value)
 358         && !isset($params[$this->_activeFields[$i]->_name])
 359         && !isset($this->_activeFields[$i]->_related)
 360       ) {
 361
 362         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 363       }
 364     }
 365     return $params;
 366   }
 367
 368   /**
 369    * Add progress bar to the import process. Calculates time remaining, status etc.
 370    *
 371    * @param $statusID
 372    *   status id of the import process saved in $config->uploadDir.
 373    * @param bool $startImport
 374    *   True when progress bar is to be initiated.
 375    * @param $startTimestamp
 376    *   Initial timestamp when the import was started.
 377    * @param $prevTimestamp
 378    *   Previous timestamp when this function was last called.
 379    * @param $totalRowCount
 380    *   Total number of rows in the import file.
 381    *
 382    * @return NULL|$currTimestamp
 383    */
 384   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 385     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 386
 387     if ($startImport) {
 388       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 389       //do not force the browser to display the save dialog, CRM-7640
 390       $contents = json_encode([0, $status]);
 391       file_put_contents($statusFile, $contents);
 392     }
 393     else {
 394       $rowCount = $this->_rowCount ?? $this->_lineCount;
 395       $currTimestamp = time();
 396       $time = ($currTimestamp - $prevTimestamp);
 397       $recordsLeft = $totalRowCount - $rowCount;
 398       if ($recordsLeft < 0) {
 399         $recordsLeft = 0;
 400       }
 401       $estimatedTime = ($recordsLeft / 50) * $time;
 402       $estMinutes = floor($estimatedTime / 60);
 403       $timeFormatted = '';
 404       if ($estMinutes > 1) {
 405         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 406         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 407       }
 408       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 409       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 410       $statusMsg = ts('%1 of %2 records - %3 remaining',
 411         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 412       );
 413       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 414       $contents = json_encode([$processedPercent, $status]);
 415
 416       file_put_contents($statusFile, $contents);
 417       return $currTimestamp;
 418     }
 419   }
 420
 421   /**
 422    * @return array
 423    */
 424   public function getSelectValues(): array {
 425     $values = [];
 426     foreach ($this->_fields as $name => $field) {
 427       $values[$name] = $field->_title;
 428     }
 429     return $values;
 430   }
 431
 432   /**
 433    * @return array
 434    */
 435   public function getSelectTypes() {
 436     $values = [];
 437     // This is only called from the MapField form in isolation now,
 438     // so we need to set the metadata.
 439     $this->init();
 440     foreach ($this->_fields as $name => $field) {
 441       if (isset($field->_hasLocationType)) {
 442         $values[$name] = $field->_hasLocationType;
 443       }
 444     }
 445     return $values;
 446   }
 447
 448   /**
 449    * @return array
 450    */
 451   public function getHeaderPatterns() {
 452     $values = [];
 453     foreach ($this->_fields as $name => $field) {
 454       if (isset($field->_headerPattern)) {
 455         $values[$name] = $field->_headerPattern;
 456       }
 457     }
 458     return $values;
 459   }
 460
 461   /**
 462    * @return array
 463    */
 464   public function getDataPatterns() {
 465     $values = [];
 466     foreach ($this->_fields as $name => $field) {
 467       $values[$name] = $field->_dataPattern;
 468     }
 469     return $values;
 470   }
 471
 472   /**
 473    * Remove single-quote enclosures from a value array (row).
 474    *
 475    * @param array $values
 476    * @param string $enclosure
 477    *
 478    * @return void
 479    */
 480   public static function encloseScrub(&$values, $enclosure = "'") {
 481     if (empty($values)) {
 482       return;
 483     }
 484
 485     foreach ($values as $k => $v) {
 486       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 487     }
 488   }
 489
 490   /**
 491    * Setter function.
 492    *
 493    * @param int $max
 494    *
 495    * @return void
 496    */
 497   public function setMaxLinesToProcess($max) {
 498     $this->_maxLinesToProcess = $max;
 499   }
 500
 501   /**
 502    * Validate that we have the required fields to create the contact or find it to update.
 503    *
 504    * Note that the users duplicate selection affects this as follows
 505    * - if they did not select an update variant then the id field is not
 506    *   permitted in the mapping - so we can assume the presence of id means
 507    *   we should use it
 508    * - the external_identifier field is valid in place of the other fields
 509    *   when they have chosen update or fill - in this case we are only looking
 510    *   to update an existing contact.
 511    *
 512    * @param string $contactType
 513    * @param array $params
 514    * @param bool $isPermitExistingMatchFields
 515    *
 516    * @return void
 517    * @throws \CRM_Core_Exception
 518    */
 519   protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE): void {
 520     if (!empty($params['id'])) {
 521       return;
 522     }
 523     $requiredFields = [
 524       'Individual' => [
 525         'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
 526         'email' => ts('Email Address'),
 527       ],
 528       'Organization' => ['organization_name' => ts('Organization Name')],
 529       'Household' => ['household_name' => ts('Household Name')],
 530     ][$contactType];
 531     if ($isPermitExistingMatchFields) {
 532       $requiredFields['external_identifier'] = ts('External Identifier');
 533       // Historically just an email has been accepted as it is 'usually good enough'
 534       // for a dedupe rule look up - but really this is a stand in for
 535       // whatever is needed to find an existing matching contact using the
 536       // specified dedupe rule (or the default Unsupervised if not specified).
 537       $requiredFields['email'] = ts('Email Address');
 538     }
 539     $this->validateRequiredFields($requiredFields, $params);
 540   }
 541
 542   /**
 543    * Determines the file extension based on error code.
 544    *
 545    * @var int $type error code constant
 546    * @return string
 547    */
 548   public static function errorFileName($type) {
 549     $fileName = NULL;
 550     if (empty($type)) {
 551       return $fileName;
 552     }
 553
 554     $config = CRM_Core_Config::singleton();
 555     $fileName = $config->uploadDir . "sqlImport";
 556     switch ($type) {
 557       case self::ERROR:
 558         $fileName .= '.errors';
 559         break;
 560
 561       case self::DUPLICATE:
 562         $fileName .= '.duplicates';
 563         break;
 564
 565       case self::NO_MATCH:
 566         $fileName .= '.mismatch';
 567         break;
 568
 569       case self::UNPARSED_ADDRESS_WARNING:
 570         $fileName .= '.unparsedAddress';
 571         break;
 572     }
 573
 574     return $fileName;
 575   }
 576
 577   /**
 578    * Determines the file name based on error code.
 579    *
 580    * @var $type error code constant
 581    * @return string
 582    */
 583   public static function saveFileName($type) {
 584     $fileName = NULL;
 585     if (empty($type)) {
 586       return $fileName;
 587     }
 588     switch ($type) {
 589       case self::ERROR:
 590         $fileName = 'Import_Errors.csv';
 591         break;
 592
 593       case self::DUPLICATE:
 594         $fileName = 'Import_Duplicates.csv';
 595         break;
 596
 597       case self::NO_MATCH:
 598         $fileName = 'Import_Mismatch.csv';
 599         break;
 600
 601       case self::UNPARSED_ADDRESS_WARNING:
 602         $fileName = 'Import_Unparsed_Address.csv';
 603         break;
 604     }
 605
 606     return $fileName;
 607   }
 608
 609   /**
 610    * Check if contact is a duplicate .
 611    *
 612    * @param array $formatValues
 613    *
 614    * @return array
 615    */
 616   protected function checkContactDuplicate(&$formatValues) {
 617     //retrieve contact id using contact dedupe rule
 618     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 619     $formatValues['version'] = 3;
 620     require_once 'CRM/Utils/DeprecatedUtils.php';
 621     $params = $formatValues;
 622     static $cIndieFields = NULL;
 623     static $defaultLocationId = NULL;
 624
 625     $contactType = $params['contact_type'];
 626     if ($cIndieFields == NULL) {
 627       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 628       $cIndieFields = $cTempIndieFields;
 629
 630       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 631
 632       // set the value to default location id else set to 1
 633       if (!$defaultLocationId = (int) $defaultLocation->id) {
 634         $defaultLocationId = 1;
 635       }
 636     }
 637
 638     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 639
 640     $contactFormatted = [];
 641     foreach ($params as $key => $field) {
 642       if ($field == NULL || $field === '') {
 643         continue;
 644       }
 645       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 646       // instead of soft credit contact.
 647       if (is_array($field) && $key != "soft_credit") {
 648         foreach ($field as $value) {
 649           $break = FALSE;
 650           if (is_array($value)) {
 651             foreach ($value as $name => $testForEmpty) {
 652               if ($name !== 'phone_type' &&
 653                 ($testForEmpty === '' || $testForEmpty == NULL)
 654               ) {
 655                 $break = TRUE;
 656                 break;
 657               }
 658             }
 659           }
 660           else {
 661             $break = TRUE;
 662           }
 663           if (!$break) {
 664             $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 665           }
 666         }
 667         continue;
 668       }
 669
 670       $value = [$key => $field];
 671
 672       // check if location related field, then we need to add primary location type
 673       if (in_array($key, $locationFields)) {
 674         $value['location_type_id'] = $defaultLocationId;
 675       }
 676       elseif (array_key_exists($key, $cIndieFields)) {
 677         $value['contact_type'] = $contactType;
 678       }
 679
 680       $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 681     }
 682
 683     $contactFormatted['contact_type'] = $contactType;
 684
 685     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 686   }
 687
 688   /**
 689    * This function adds the contact variable in $values to the
 690    * parameter list $params.  For most cases, $values should have length 1.  If
 691    * the variable being added is a child of Location, a location_type_id must
 692    * also be included.  If it is a child of phone, a phone_type must be included.
 693    *
 694    * @param array $values
 695    *   The variable(s) to be added.
 696    * @param array $params
 697    *   The structured parameter list.
 698    *
 699    * @return bool|CRM_Utils_Error
 700    */
 701   private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
 702     // @todo - like most functions in import ... most of this is cruft....
 703     // Crawl through the possible classes:
 704     // Contact
 705     //      Individual
 706     //      Household
 707     //      Organization
 708     //          Location
 709     //              Address
 710     //              Email
 711     //              Phone
 712     //              IM
 713     //      Note
 714     //      Custom
 715
 716     // Cache the various object fields
 717     static $fields = NULL;
 718
 719     if ($fields == NULL) {
 720       $fields = [];
 721     }
 722
 723     // first add core contact values since for other Civi modules they are not added
 724     require_once 'CRM/Contact/BAO/Contact.php';
 725     $contactFields = CRM_Contact_DAO_Contact::fields();
 726     _civicrm_api3_store_values($contactFields, $values, $params);
 727
 728     if (isset($values['contact_type'])) {
 729       // we're an individual/household/org property
 730
 731       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 732
 733       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 734       return TRUE;
 735     }
 736
 737     if (isset($values['individual_prefix'])) {
 738       if (!empty($params['prefix_id'])) {
 739         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 740         $params['prefix'] = $prefixes[$params['prefix_id']];
 741       }
 742       else {
 743         $params['prefix'] = $values['individual_prefix'];
 744       }
 745       return TRUE;
 746     }
 747
 748     if (isset($values['individual_suffix'])) {
 749       if (!empty($params['suffix_id'])) {
 750         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
 751         $params['suffix'] = $suffixes[$params['suffix_id']];
 752       }
 753       else {
 754         $params['suffix'] = $values['individual_suffix'];
 755       }
 756       return TRUE;
 757     }
 758
 759     // CRM-4575
 760     if (isset($values['email_greeting'])) {
 761       if (!empty($params['email_greeting_id'])) {
 762         $emailGreetingFilter = [
 763           'contact_type' => $params['contact_type'] ?? NULL,
 764           'greeting_type' => 'email_greeting',
 765         ];
 766         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
 767         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
 768       }
 769       else {
 770         $params['email_greeting'] = $values['email_greeting'];
 771       }
 772
 773       return TRUE;
 774     }
 775
 776     if (isset($values['postal_greeting'])) {
 777       if (!empty($params['postal_greeting_id'])) {
 778         $postalGreetingFilter = [
 779           'contact_type' => $params['contact_type'] ?? NULL,
 780           'greeting_type' => 'postal_greeting',
 781         ];
 782         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
 783         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
 784       }
 785       else {
 786         $params['postal_greeting'] = $values['postal_greeting'];
 787       }
 788       return TRUE;
 789     }
 790
 791     if (isset($values['addressee'])) {
 792       $params['addressee'] = $values['addressee'];
 793       return TRUE;
 794     }
 795
 796     if (isset($values['gender'])) {
 797       if (!empty($params['gender_id'])) {
 798         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
 799         $params['gender'] = $genders[$params['gender_id']];
 800       }
 801       else {
 802         $params['gender'] = $values['gender'];
 803       }
 804       return TRUE;
 805     }
 806
 807     if (!empty($values['preferred_communication_method'])) {
 808       $comm = [];
 809       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
 810
 811       $preffComm = explode(',', $values['preferred_communication_method']);
 812       foreach ($preffComm as $v) {
 813         $v = strtolower(trim($v));
 814         if (array_key_exists($v, $pcm)) {
 815           $comm[$pcm[$v]] = 1;
 816         }
 817       }
 818
 819       $params['preferred_communication_method'] = $comm;
 820       return TRUE;
 821     }
 822
 823     // format the website params.
 824     if (!empty($values['url'])) {
 825       static $websiteFields;
 826       if (!is_array($websiteFields)) {
 827         require_once 'CRM/Core/DAO/Website.php';
 828         $websiteFields = CRM_Core_DAO_Website::fields();
 829       }
 830       if (!array_key_exists('website', $params) ||
 831         !is_array($params['website'])
 832       ) {
 833         $params['website'] = [];
 834       }
 835
 836       $websiteCount = count($params['website']);
 837       _civicrm_api3_store_values($websiteFields, $values,
 838         $params['website'][++$websiteCount]
 839       );
 840
 841       return TRUE;
 842     }
 843
 844     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
 845     if (!empty($values['location_type_id'])) {
 846       static $fields = NULL;
 847       if ($fields == NULL) {
 848         $fields = [];
 849       }
 850
 851       foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
 852         $name = strtolower($block);
 853         if (!array_key_exists($name, $values)) {
 854           continue;
 855         }
 856
 857         if ($name === 'phone_ext') {
 858           $block = 'Phone';
 859         }
 860
 861         // block present in value array.
 862         if (!array_key_exists($name, $params) || !is_array($params[$name])) {
 863           $params[$name] = [];
 864         }
 865
 866         if (!array_key_exists($block, $fields)) {
 867           $className = "CRM_Core_DAO_$block";
 868           $fields[$block] =& $className::fields();
 869         }
 870
 871         $blockCnt = count($params[$name]);
 872
 873         // copy value to dao field name.
 874         if ($name == 'im') {
 875           $values['name'] = $values[$name];
 876         }
 877
 878         _civicrm_api3_store_values($fields[$block], $values,
 879           $params[$name][++$blockCnt]
 880         );
 881
 882         if (empty($params['id']) && ($blockCnt == 1)) {
 883           $params[$name][$blockCnt]['is_primary'] = TRUE;
 884         }
 885
 886         // we only process single block at a time.
 887         return TRUE;
 888       }
 889
 890       // handle address fields.
 891       if (!array_key_exists('address', $params) || !is_array($params['address'])) {
 892         $params['address'] = [];
 893       }
 894
 895       $addressCnt = 1;
 896       foreach ($params['address'] as $cnt => $addressBlock) {
 897         if (CRM_Utils_Array::value('location_type_id', $values) ==
 898           CRM_Utils_Array::value('location_type_id', $addressBlock)
 899         ) {
 900           $addressCnt = $cnt;
 901           break;
 902         }
 903         $addressCnt++;
 904       }
 905
 906       if (!array_key_exists('Address', $fields)) {
 907         $fields['Address'] = CRM_Core_DAO_Address::fields();
 908       }
 909
 910       // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
 911       // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
 912       // the address in CRM_Core_BAO_Address::create method
 913       if (!empty($values['location_type_id'])) {
 914         static $customFields = [];
 915         if (empty($customFields)) {
 916           $customFields = CRM_Core_BAO_CustomField::getFields('Address');
 917         }
 918         // make a copy of values, as we going to make changes
 919         $newValues = $values;
 920         foreach ($values as $key => $val) {
 921           $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 922           if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
 923             // mark an entry in fields array since we want the value of custom field to be copied
 924             $fields['Address'][$key] = NULL;
 925
 926             $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 927             if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
 928               $mulValues = explode(',', $val);
 929               $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 930               $newValues[$key] = [];
 931               foreach ($mulValues as $v1) {
 932                 foreach ($customOption as $v2) {
 933                   if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 934                     (strtolower($v2['value']) == strtolower(trim($v1)))
 935                   ) {
 936                     if ($htmlType == 'CheckBox') {
 937                       $newValues[$key][$v2['value']] = 1;
 938                     }
 939                     else {
 940                       $newValues[$key][] = $v2['value'];
 941                     }
 942                   }
 943                 }
 944               }
 945             }
 946           }
 947         }
 948         // consider new values
 949         $values = $newValues;
 950       }
 951
 952       _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
 953
 954       $addressFields = [
 955         'county',
 956         'country',
 957         'state_province',
 958         'supplemental_address_1',
 959         'supplemental_address_2',
 960         'supplemental_address_3',
 961         'StateProvince.name',
 962       ];
 963
 964       foreach ($addressFields as $field) {
 965         if (array_key_exists($field, $values)) {
 966           if (!array_key_exists('address', $params)) {
 967             $params['address'] = [];
 968           }
 969           $params['address'][$addressCnt][$field] = $values[$field];
 970         }
 971       }
 972
 973       if ($addressCnt == 1) {
 974
 975         $params['address'][$addressCnt]['is_primary'] = TRUE;
 976       }
 977       return TRUE;
 978     }
 979
 980     if (isset($values['note'])) {
 981       // add a note field
 982       if (!isset($params['note'])) {
 983         $params['note'] = [];
 984       }
 985       $noteBlock = count($params['note']) + 1;
 986
 987       $params['note'][$noteBlock] = [];
 988       if (!isset($fields['Note'])) {
 989         $fields['Note'] = CRM_Core_DAO_Note::fields();
 990       }
 991
 992       // get the current logged in civicrm user
 993       $session = CRM_Core_Session::singleton();
 994       $userID = $session->get('userID');
 995
 996       if ($userID) {
 997         $values['contact_id'] = $userID;
 998       }
 999
1000       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1001
1002       return TRUE;
1003     }
1004
1005     // Check for custom field values
1006
1007     if (empty($fields['custom'])) {
1008       $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1009         FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1010       );
1011     }
1012
1013     foreach ($values as $key => $value) {
1014       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1015         // check if it's a valid custom field id
1016
1017         if (!array_key_exists($customFieldID, $fields['custom'])) {
1018           return civicrm_api3_create_error('Invalid custom field ID');
1019         }
1020         else {
1021           $params[$key] = $value;
1022         }
1023       }
1024     }
1025   }
1026
1027   /**
1028    * Parse a field which could be represented by a label or name value rather than the DB value.
1029    *
1030    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1031    *
1032    * but if not available then see if we have a label that can be converted to a name.
1033    *
1034    * @param string|int|null $submittedValue
1035    * @param array $fieldSpec
1036    *   Metadata for the field
1037    *
1038    * @return mixed
1039    */
1040   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1041     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1042     if (!isset($fieldSpec['bao'])) {
1043       return $submittedValue;
1044     }
1045     /* @var \CRM_Core_DAO $bao */
1046     $bao = $fieldSpec['bao'];
1047     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1048     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1049     if (isset($nameOptions[$submittedValue])) {
1050       return $submittedValue;
1051     }
1052     if (in_array($submittedValue, $nameOptions)) {
1053       return array_search($submittedValue, $nameOptions, TRUE);
1054     }
1055
1056     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1057     if (isset($labelOptions[$submittedValue])) {
1058       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1059     }
1060     return '';
1061   }
1062
1063   /**
1064    * This is code extracted from 4 places where this exact snippet was being duplicated.
1065    *
1066    * FIXME: Extracting this was a first step, but there's also
1067    *  1. Inconsistency in the way other select options are handled.
1068    *     Contribution adds handling for Select/Radio/Autocomplete
1069    *     Participant/Activity only handles Select/Radio and misses Autocomplete
1070    *     Membership is missing all of it
1071    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1072    *
1073    * @param $customFieldID
1074    * @param $value
1075    * @param $fieldType
1076    * @return array
1077    */
1078   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1079     $mulValues = explode(',', $value);
1080     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1081     $values = [];
1082     foreach ($mulValues as $v1) {
1083       foreach ($customOption as $customValueID => $customLabel) {
1084         $customValue = $customLabel['value'];
1085         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1086           (strtolower(trim($customValue)) == strtolower(trim($v1)))
1087         ) {
1088           $values[] = $customValue;
1089         }
1090       }
1091     }
1092     return $values;
1093   }
1094
1095   /**
1096    * Get the ids of any contacts that match according to the rule.
1097    *
1098    * @param array $formatted
1099    *
1100    * @return array
1101    */
1102   protected function getIdsOfMatchingContacts(array $formatted):array {
1103     // the call to the deprecated function seems to add no value other that to do an additional
1104     // check for the contact_id & type.
1105     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1106     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
1107       return [];
1108     }
1109     if (is_array($error['error_message']['params'][0])) {
1110       return $error['error_message']['params'][0];
1111     }
1112     else {
1113       return explode(',', $error['error_message']['params'][0]);
1114     }
1115   }
1116
1117   /**
1118    * Validate that the field requirements are met in the params.
1119    *
1120    * @param array $requiredFields
1121    * @param array $params
1122    *   An array of required fields (fieldName => label)
1123    *   - note this follows the and / or array nesting we see in permission checks
1124    *   eg.
1125    *   [
1126    *     'email' => ts('Email'),
1127    *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1128    *   ]
1129    *   Means 'email' OR 'first_name AND 'last_name'.
1130    *
1131    * @throws \CRM_Core_Exception
1132    *   Exception thrown if field requirements are not met.
1133    */
1134   protected function validateRequiredFields(array $requiredFields, array $params): void {
1135     $missingFields = [];
1136     foreach ($requiredFields as $key => $required) {
1137       if (!is_array($required)) {
1138         $importParameter = $params[$key] ?? [];
1139         if (!is_array($importParameter)) {
1140           if (!empty($importParameter)) {
1141             return;
1142           }
1143         }
1144         else {
1145           foreach ($importParameter as $locationValues) {
1146             if (!empty($locationValues[$key])) {
1147               return;
1148             }
1149           }
1150         }
1151
1152         $missingFields[$key] = $required;
1153       }
1154       else {
1155         foreach ($required as $field => $label) {
1156           if (empty($params[$field])) {
1157             $missing[$field] = $label;
1158           }
1159         }
1160         if (empty($missing)) {
1161           return;
1162         }
1163         $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1164       }
1165     }
1166     throw new CRM_Core_Exception(ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1167   }
1168
1169 }