CRM/Contact/Import/Parser/Contact.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Contact;
  13 use Civi\Api4\RelationshipType;
  14
  15 require_once 'CRM/Utils/DeprecatedUtils.php';
  16 require_once 'api/v3/utils.php';
  17
  18 /**
  19  *
  20  * @package CRM
  21  * @copyright CiviCRM LLC https://civicrm.org/licensing
  22  */
  23
  24 /**
  25  * class to parse contact csv files
  26  */
  27 class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser {
  28
  29   use CRM_Contact_Import_MetadataTrait;
  30
  31   protected $_mapperKeys = [];
  32   protected $_relationships;
  33
  34   /**
  35    * Is update only permitted on an id match.
  36    *
  37    * Note this historically was true for when id or external identifier was
  38    * present. However, CRM-17275 determined that a dedupe-match could over-ride
  39    * external identifier.
  40    *
  41    * @var bool
  42    */
  43   protected $_updateWithId;
  44   protected $_retCode;
  45
  46   protected $_externalIdentifierIndex;
  47   protected $_allExternalIdentifiers = [];
  48   protected $_parseStreetAddress;
  49
  50   /**
  51    * Array of successfully imported contact id's
  52    *
  53    * @var array
  54    */
  55   protected $_newContacts;
  56
  57   /**
  58    * Line count id.
  59    *
  60    * @var int
  61    */
  62   protected $_lineCount;
  63
  64   /**
  65    * Array of successfully imported related contact id's
  66    *
  67    * @var array
  68    */
  69   protected $_newRelatedContacts;
  70
  71   protected $_tableName;
  72
  73   /**
  74    * Total number of lines in file
  75    *
  76    * @var int
  77    */
  78   protected $_rowCount;
  79
  80   protected $_primaryKeyName;
  81   protected $_statusFieldName;
  82
  83   protected $fieldMetadata = [];
  84
  85   /**
  86    * Relationship labels.
  87    *
  88    * Temporary cache of labels to reduce queries in getRelationshipLabels.
  89    *
  90    * @var array
  91    *   e.g ['5a_b' => 'Employer', '5b_a' => 'Employee']
  92    */
  93   protected $relationshipLabels = [];
  94
  95   /**
  96    * On duplicate
  97    *
  98    * @var int
  99    */
 100   public $_onDuplicate;
 101
 102   /**
 103    * Dedupe rule group id to use if set
 104    *
 105    * @var int
 106    */
 107   public $_dedupeRuleGroupID = NULL;
 108
 109   /**
 110    * Class constructor.
 111    *
 112    * @param array $mapperKeys
 113    */
 114   public function __construct($mapperKeys = []) {
 115     parent::__construct();
 116     $this->_mapperKeys = $mapperKeys;
 117   }
 118
 119   /**
 120    * The initializer code, called before processing.
 121    */
 122   public function init() {
 123     $this->setFieldMetadata();
 124     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 125       $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field));
 126     }
 127     $this->_newContacts = [];
 128
 129     $this->setActiveFields($this->_mapperKeys);
 130
 131     $this->_externalIdentifierIndex = -1;
 132
 133     $index = 0;
 134     foreach ($this->_mapperKeys as $key) {
 135       if ($key == 'external_identifier') {
 136         $this->_externalIdentifierIndex = $index;
 137       }
 138       $index++;
 139     }
 140
 141     $this->_updateWithId = FALSE;
 142     if (in_array('id', $this->_mapperKeys) || ($this->_externalIdentifierIndex >= 0 && $this->isUpdateExistingContacts())) {
 143       $this->_updateWithId = TRUE;
 144     }
 145
 146     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
 147   }
 148
 149   /**
 150    * Is this a case where the user has opted to update existing contacts.
 151    *
 152    * @return bool
 153    *
 154    * @throws \API_Exception
 155    */
 156   private function isUpdateExistingContacts(): bool {
 157     return in_array((int) $this->getSubmittedValue('onDuplicate'), [
 158       CRM_Import_Parser::DUPLICATE_UPDATE,
 159       CRM_Import_Parser::DUPLICATE_FILL,
 160     ], TRUE);
 161   }
 162
 163   /**
 164    * Gets the fields available for importing in a key-name, title format.
 165    *
 166    * @return array
 167    *   eg. ['first_name' => 'First Name'.....]
 168    *
 169    * @throws \API_Exception
 170    *
 171    * @todo - we are constructing the metadata before we
 172    * have set the contact type so we re-do it here.
 173    *
 174    * Once we have cleaned up the way the mapper is handled
 175    * we can ditch all the existing _construct parameters in favour
 176    * of just the userJobID - there are current open PRs towards this end.
 177    */
 178   public function getAvailableFields(): array {
 179     $this->setFieldMetadata();
 180     $return = [];
 181     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 182       if ($name === 'id' && $this->isSkipDuplicates()) {
 183         // Duplicates are being skipped so id matching is not availble.
 184         continue;
 185       }
 186       $return[$name] = $field['title'];
 187     }
 188     return $return;
 189   }
 190
 191   /**
 192    * Did the user specify duplicates should be skipped and not imported.
 193    *
 194    * @return bool
 195    *
 196    * @throws \API_Exception
 197    */
 198   private function isSkipDuplicates(): bool {
 199     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 200   }
 201
 202   /**
 203    * Handle the values in preview mode.
 204    *
 205    * Function will be deprecated in favour of validateValues.
 206    *
 207    * @param array $values
 208    *   The array of values belonging to this line.
 209    *
 210    * @return bool
 211    *   the result of this processing
 212    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 213    */
 214   public function preview(&$values) {
 215     return $this->summary($values);
 216   }
 217
 218   /**
 219    * Handle the values in summary mode.
 220    *
 221    * Function will be deprecated in favour of validateValues.
 222    *
 223    * @param array $values
 224    *   The array of values belonging to this line.
 225    *
 226    * @return int
 227    *   the result of this processing
 228    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 229    */
 230   public function summary(&$values): int {
 231     $rowNumber = (int) ($values[count($values) - 1]);
 232     try {
 233       $this->validateValues($values);
 234     }
 235     catch (CRM_Core_Exception $e) {
 236       $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
 237       array_unshift($values, $e->getMessage());
 238       return CRM_Import_Parser::ERROR;
 239     }
 240     $this->setImportStatus($rowNumber, 'NEW', '');
 241
 242     return CRM_Import_Parser::VALID;
 243   }
 244
 245   /**
 246    * Get Array of all the fields that could potentially be part
 247    * import process
 248    *
 249    * @return array
 250    */
 251   public function getAllFields() {
 252     return $this->_fields;
 253   }
 254
 255   /**
 256    * Handle the values in import mode.
 257    *
 258    * @param int $onDuplicate
 259    *   The code for what action to take on duplicates.
 260    * @param array $values
 261    *   The array of values belonging to this line.
 262    *
 263    * @return bool
 264    *   the result of this processing
 265    *
 266    * @throws \CiviCRM_API3_Exception
 267    * @throws \CRM_Core_Exception
 268    * @throws \API_Exception
 269    */
 270   public function import($onDuplicate, &$values) {
 271     $this->_unparsedStreetAddressContacts = [];
 272     if (!$this->getSubmittedValue('doGeocodeAddress')) {
 273       // CRM-5854, reset the geocode method to null to prevent geocoding
 274       CRM_Utils_GeocodeProvider::disableForSession();
 275     }
 276
 277     // first make sure this is a valid line
 278     //$this->_updateWithId = false;
 279     $response = $this->summary($values);
 280     $statusFieldName = $this->_statusFieldName;
 281
 282     if ($response != CRM_Import_Parser::VALID) {
 283       $this->setImportStatus((int) $values[count($values) - 1], 'Invalid', "Invalid (Error Code: $response)");
 284       return $response;
 285     }
 286
 287     $params = $this->getMappedRow($values);
 288     $formatted = [
 289       'contact_type' => $this->_contactType,
 290     ];
 291
 292     $contactFields = CRM_Contact_DAO_Contact::import();
 293
 294     //check if external identifier exists in database
 295     if (!empty($params['external_identifier']) && (!empty($params['id']) || in_array($onDuplicate, [
 296       CRM_Import_Parser::DUPLICATE_SKIP,
 297       CRM_Import_Parser::DUPLICATE_NOCHECK,
 298     ]))) {
 299
 300       $extIDResult = civicrm_api3('Contact', 'get', [
 301         'external_identifier' => $params['external_identifier'],
 302         'showAll' => 'all',
 303         'return' => ['id', 'contact_is_deleted'],
 304       ]);
 305       if (isset($extIDResult['id'])) {
 306         // record with matching external identifier does exist.
 307         $internalCid = $extIDResult['id'];
 308         if ($internalCid != CRM_Utils_Array::value('id', $params)) {
 309           if ($extIDResult['values'][$internalCid]['contact_is_deleted'] == 1) {
 310             // And it is deleted. What to do? If we skip it, they user
 311             // will be under the impression that the record exists in
 312             // the database, yet they won't be able to find it. If we
 313             // don't skip it, the database will try to insert a new record
 314             // with an external_identifier that is non-unique. So...
 315             // we will update this contact to remove the external_identifier
 316             // and let a new record be created.
 317             $update_params = ['id' => $internalCid, 'external_identifier' => ''];
 318             civicrm_api3('Contact', 'create', $update_params);
 319           }
 320           else {
 321             $errorMessage = ts('External ID already exists in Database.');
 322             array_unshift($values, $errorMessage);
 323             $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 324             return CRM_Import_Parser::DUPLICATE;
 325           }
 326         }
 327       }
 328     }
 329
 330     if (!empty($this->_contactSubType)) {
 331       $params['contact_sub_type'] = $this->_contactSubType;
 332     }
 333
 334     if ($subType = CRM_Utils_Array::value('contact_sub_type', $params)) {
 335       if (CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType, FALSE, 'label')) {
 336         $subTypes = CRM_Contact_BAO_ContactType::subTypePairs($this->_contactType, FALSE, NULL);
 337         $params['contact_sub_type'] = array_search($subType, $subTypes);
 338       }
 339       elseif (!CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType)) {
 340         $message = "Mismatched or Invalid Contact Subtype.";
 341         array_unshift($values, $message);
 342         return CRM_Import_Parser::NO_MATCH;
 343       }
 344     }
 345
 346     // Get contact id to format common data in update/fill mode,
 347     // prioritising a dedupe rule check over an external_identifier check, but falling back on ext id.
 348     if ($this->_updateWithId && empty($params['id'])) {
 349       try {
 350         $possibleMatches = $this->getPossibleContactMatches($params);
 351       }
 352       catch (CRM_Core_Exception $e) {
 353         $errorMessage = $e->getMessage();
 354         array_unshift($values, $errorMessage);
 355
 356         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 357         return CRM_Import_Parser::ERROR;
 358       }
 359       foreach ($possibleMatches as $possibleID) {
 360         $params['id'] = $formatted['id'] = $possibleID;
 361       }
 362     }
 363     //format common data, CRM-4062
 364     $this->formatCommonData($params, $formatted, $contactFields);
 365
 366     $relationship = FALSE;
 367     $createNewContact = TRUE;
 368     // Support Match and Update Via Contact ID
 369     if ($this->_updateWithId && isset($params['id'])) {
 370       $createNewContact = FALSE;
 371       // @todo - it feels like all the rows from here to the end of the IF
 372       // could be removed in favour of a simple check for whether the contact_type & id match
 373       $matchedIDs = $this->getIdsOfMatchingContacts($formatted);
 374       if (!empty($matchedIDs)) {
 375         if (count($matchedIDs) >= 1) {
 376           $updateflag = TRUE;
 377           foreach ($matchedIDs as $contactId) {
 378             if ($params['id'] == $contactId) {
 379               $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 380               if ($formatted['contact_type'] == $contactType) {
 381                 //validation of subtype for update mode
 382                 //CRM-5125
 383                 $contactSubType = NULL;
 384                 if (!empty($params['contact_sub_type'])) {
 385                   $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 386                 }
 387
 388                 if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 389
 390                   $message = "Mismatched contact SubTypes :";
 391                   array_unshift($values, $message);
 392                   $updateflag = FALSE;
 393                   $this->_retCode = CRM_Import_Parser::NO_MATCH;
 394                 }
 395                 else {
 396                   $updateflag = FALSE;
 397                   $this->_retCode = CRM_Import_Parser::VALID;
 398                 }
 399               }
 400               else {
 401                 $message = "Mismatched contact Types :";
 402                 array_unshift($values, $message);
 403                 $updateflag = FALSE;
 404                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 405               }
 406             }
 407           }
 408           if ($updateflag) {
 409             $message = "Mismatched contact IDs OR Mismatched contact Types :";
 410             array_unshift($values, $message);
 411             $this->_retCode = CRM_Import_Parser::NO_MATCH;
 412           }
 413         }
 414       }
 415       else {
 416         $contactType = NULL;
 417         if (!empty($params['id'])) {
 418           $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 419           if ($contactType) {
 420             if ($formatted['contact_type'] == $contactType) {
 421               //validation of subtype for update mode
 422               //CRM-5125
 423               $contactSubType = NULL;
 424               if (!empty($params['contact_sub_type'])) {
 425                 $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 426               }
 427
 428               if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 429
 430                 $message = "Mismatched contact SubTypes :";
 431                 array_unshift($values, $message);
 432                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 433               }
 434               else {
 435                 $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $params['id'], FALSE, $this->_dedupeRuleGroupID);
 436                 $this->_retCode = CRM_Import_Parser::VALID;
 437               }
 438             }
 439             else {
 440               $message = "Mismatched contact Types :";
 441               array_unshift($values, $message);
 442               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 443             }
 444           }
 445           else {
 446             // we should avoid multiple errors for single record
 447             // since we have already retCode and we trying to force again.
 448             if ($this->_retCode != CRM_Import_Parser::NO_MATCH) {
 449               $message = "No contact found for this contact ID:" . $params['id'];
 450               array_unshift($values, $message);
 451               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 452             }
 453           }
 454         }
 455         else {
 456           //CRM-4148
 457           //now we want to create new contact on update/fill also.
 458           $createNewContact = TRUE;
 459         }
 460       }
 461
 462       if (isset($newContact) && is_a($newContact, 'CRM_Contact_BAO_Contact')) {
 463         $relationship = TRUE;
 464       }
 465     }
 466
 467     //fixed CRM-4148
 468     //now we create new contact in update/fill mode also.
 469     $contactID = NULL;
 470     if ($createNewContact || ($this->_retCode != CRM_Import_Parser::NO_MATCH && $this->_updateWithId)) {
 471       // @todo - there are multiple places where formatting is done that need consolidation.
 472       // This handles where the label has been passed in and it has gotten this far.
 473       // probably a bunch of hard-coded stuff could be removed to rely on this.
 474       $fields = Contact::getFields(FALSE)
 475         ->addWhere('options', '=', TRUE)
 476         ->setLoadOptions(TRUE)
 477         ->execute()->indexBy('name');
 478       foreach ($fields as $fieldName => $fieldSpec) {
 479         if (isset($formatted[$fieldName]) && is_array($formatted[$fieldName])) {
 480           // If we have an array at this stage, it's probably a multi-select
 481           // field that has already been parsed properly into the value that
 482           // should be inserted into the database.
 483           continue;
 484         }
 485         if (!empty($formatted[$fieldName])
 486           && empty($fieldSpec['options'][$formatted[$fieldName]])) {
 487           $formatted[$fieldName] = array_search($formatted[$fieldName], $fieldSpec['options'], TRUE) ?? $formatted[$fieldName];
 488         }
 489       }
 490       //CRM-4430, don't carry if not submitted.
 491       if ($this->_updateWithId && !empty($params['id'])) {
 492         $contactID = $params['id'];
 493       }
 494       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactID, TRUE, $this->_dedupeRuleGroupID);
 495     }
 496
 497     if (isset($newContact) && is_object($newContact) && ($newContact instanceof CRM_Contact_BAO_Contact)) {
 498       $relationship = TRUE;
 499       $newContact = clone($newContact);
 500       $contactID = $newContact->id;
 501       $this->_newContacts[] = $contactID;
 502
 503       //get return code if we create new contact in update mode, CRM-4148
 504       if ($this->_updateWithId) {
 505         $this->_retCode = CRM_Import_Parser::VALID;
 506       }
 507     }
 508     elseif (isset($newContact) && CRM_Core_Error::isAPIError($newContact, CRM_Core_Error::DUPLICATE_CONTACT)) {
 509       // if duplicate, no need of further processing
 510       if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
 511         $errorMessage = "Skipping duplicate record";
 512         array_unshift($values, $errorMessage);
 513         $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', $errorMessage);
 514         return CRM_Import_Parser::DUPLICATE;
 515       }
 516
 517       $relationship = TRUE;
 518       // CRM-10433/CRM-20739 - IDs could be string or array; handle accordingly
 519       if (!is_array($dupeContactIDs = $newContact['error_message']['params'][0])) {
 520         $dupeContactIDs = explode(',', $dupeContactIDs);
 521       }
 522       $dupeCount = count($dupeContactIDs);
 523       $contactID = array_pop($dupeContactIDs);
 524       // check to see if we had more than one duplicate contact id.
 525       // if we have more than one, the record will be rejected below
 526       if ($dupeCount == 1) {
 527         // there was only one dupe, we will continue normally...
 528         if (!in_array($contactID, $this->_newContacts)) {
 529           $this->_newContacts[] = $contactID;
 530         }
 531       }
 532     }
 533
 534     if ($contactID) {
 535       // call import hook
 536       $currentImportID = end($values);
 537
 538       $hookParams = [
 539         'contactID' => $contactID,
 540         'importID' => $currentImportID,
 541         'importTempTable' => $this->_tableName,
 542         'fieldHeaders' => $this->_mapperKeys,
 543         'fields' => $this->_activeFields,
 544       ];
 545
 546       CRM_Utils_Hook::import('Contact', 'process', $this, $hookParams);
 547     }
 548
 549     if ($relationship) {
 550       $primaryContactId = NULL;
 551       if (CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 552         if ($dupeCount == 1 && CRM_Utils_Rule::integer($contactID)) {
 553           $primaryContactId = $contactID;
 554         }
 555       }
 556       else {
 557         $primaryContactId = $newContact->id;
 558       }
 559
 560       if ((CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || is_a($newContact, 'CRM_Contact_BAO_Contact')) && $primaryContactId) {
 561
 562         //relationship contact insert
 563         foreach ($params as $key => $field) {
 564           [$id, $first, $second] = CRM_Utils_System::explode('_', $key, 3);
 565           if (!($first == 'a' && $second == 'b') && !($first == 'b' && $second == 'a')) {
 566             continue;
 567           }
 568
 569           $relationType = new CRM_Contact_DAO_RelationshipType();
 570           $relationType->id = $id;
 571           $relationType->find(TRUE);
 572           $direction = "contact_sub_type_$second";
 573
 574           $formatting = [
 575             'contact_type' => $params[$key]['contact_type'],
 576           ];
 577
 578           //set subtype for related contact CRM-5125
 579           if (isset($relationType->$direction)) {
 580             //validation of related contact subtype for update mode
 581             if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $params[$key]) && $relCsType != $relationType->$direction) {
 582               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 583               array_unshift($values, $errorMessage);
 584               return CRM_Import_Parser::NO_MATCH;
 585             }
 586             else {
 587               $formatting['contact_sub_type'] = $relationType->$direction;
 588             }
 589           }
 590
 591           $contactFields = NULL;
 592           $contactFields = CRM_Contact_DAO_Contact::import();
 593
 594           //Relation on the basis of External Identifier.
 595           if (empty($params[$key]['id']) && !empty($params[$key]['external_identifier'])) {
 596             $params[$key]['id'] = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['external_identifier'], 'id', 'external_identifier');
 597           }
 598           // check for valid related contact id in update/fill mode, CRM-4424
 599           if (in_array($onDuplicate, [
 600             CRM_Import_Parser::DUPLICATE_UPDATE,
 601             CRM_Import_Parser::DUPLICATE_FILL,
 602           ]) && !empty($params[$key]['id'])) {
 603             $relatedContactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_type');
 604             if (!$relatedContactType) {
 605               $errorMessage = ts("No contact found for this related contact ID: %1", [1 => $params[$key]['id']]);
 606               array_unshift($values, $errorMessage);
 607               return CRM_Import_Parser::NO_MATCH;
 608             }
 609
 610             //validation of related contact subtype for update mode
 611             //CRM-5125
 612             $relatedCsType = NULL;
 613             if (!empty($formatting['contact_sub_type'])) {
 614               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_sub_type');
 615             }
 616
 617             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params[$key]['id'], $relatedCsType) &&
 618                 $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))
 619             ) {
 620               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.") . ' ' . ts("ID: %1", [1 => $params[$key]['id']]);
 621               array_unshift($values, $errorMessage);
 622               return CRM_Import_Parser::NO_MATCH;
 623             }
 624             // get related contact id to format data in update/fill mode,
 625             //if external identifier is present, CRM-4423
 626             $formatting['id'] = $params[$key]['id'];
 627           }
 628
 629           //format common data, CRM-4062
 630           $this->formatCommonData($field, $formatting, $contactFields);
 631
 632           //fixed for CRM-4148
 633           if (!empty($params[$key]['id'])) {
 634             $contact = [
 635               'contact_id' => $params[$key]['id'],
 636             ];
 637             $defaults = [];
 638             $relatedNewContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
 639           }
 640           else {
 641             $relatedNewContact = $this->createContact($formatting, $contactFields, $onDuplicate, NULL, FALSE);
 642           }
 643
 644           if (is_object($relatedNewContact) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 645             $relatedNewContact = clone($relatedNewContact);
 646           }
 647
 648           $matchedIDs = [];
 649           // To update/fill contact, get the matching contact Ids if duplicate contact found
 650           // otherwise get contact Id from object of related contact
 651           if (is_array($relatedNewContact) && civicrm_error($relatedNewContact)) {
 652             if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 653               $matchedIDs = $relatedNewContact['error_message']['params'][0];
 654               if (!is_array($matchedIDs)) {
 655                 $matchedIDs = explode(',', $matchedIDs);
 656               }
 657             }
 658             else {
 659               $errorMessage = $relatedNewContact['error_message'];
 660               array_unshift($values, $errorMessage);
 661               $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 662               return CRM_Import_Parser::ERROR;
 663             }
 664           }
 665           else {
 666             $matchedIDs[] = $relatedNewContact->id;
 667           }
 668           // update/fill related contact after getting matching Contact Ids, CRM-4424
 669           if (in_array($onDuplicate, [
 670             CRM_Import_Parser::DUPLICATE_UPDATE,
 671             CRM_Import_Parser::DUPLICATE_FILL,
 672           ])) {
 673             //validation of related contact subtype for update mode
 674             //CRM-5125
 675             $relatedCsType = NULL;
 676             if (!empty($formatting['contact_sub_type'])) {
 677               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $matchedIDs[0], 'contact_sub_type');
 678             }
 679
 680             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($matchedIDs[0], $relatedCsType) && $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))) {
 681               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 682               array_unshift($values, $errorMessage);
 683               return CRM_Import_Parser::NO_MATCH;
 684             }
 685             else {
 686               $updatedContact = $this->createContact($formatting, $contactFields, $onDuplicate, $matchedIDs[0]);
 687             }
 688           }
 689           static $relativeContact = [];
 690           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 691             if (count($matchedIDs) >= 1) {
 692               $relContactId = $matchedIDs[0];
 693               //add relative contact to count during update & fill mode.
 694               //logic to make count distinct by contact id.
 695               if ($this->_newRelatedContacts || !empty($relativeContact)) {
 696                 $reContact = array_keys($relativeContact, $relContactId);
 697
 698                 if (empty($reContact)) {
 699                   $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 700                 }
 701               }
 702               else {
 703                 $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 704               }
 705             }
 706           }
 707           else {
 708             $relContactId = $relatedNewContact->id;
 709             $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 710           }
 711
 712           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 713             //fix for CRM-1993.Checks for duplicate related contacts
 714             if (count($matchedIDs) >= 1) {
 715               //if more than one duplicate contact
 716               //found, create relationship with first contact
 717               // now create the relationship record
 718               $relationParams = [
 719                 'relationship_type_id' => $key,
 720                 'contact_check' => [
 721                   $relContactId => 1,
 722                 ],
 723                 'is_active' => 1,
 724                 'skipRecentView' => TRUE,
 725               ];
 726
 727               // we only handle related contact success, we ignore failures for now
 728               // at some point wold be nice to have related counts as separate
 729               $relationIds = [
 730                 'contact' => $primaryContactId,
 731               ];
 732
 733               [$valid, $duplicate] = self::legacyCreateMultiple($relationParams, $relationIds);
 734
 735               if ($valid || $duplicate) {
 736                 $relationIds['contactTarget'] = $relContactId;
 737                 $action = ($duplicate) ? CRM_Core_Action::UPDATE : CRM_Core_Action::ADD;
 738                 CRM_Contact_BAO_Relationship::relatedMemberships($primaryContactId, $relationParams, $relationIds, $action);
 739               }
 740
 741               //handle current employer, CRM-3532
 742               if ($valid) {
 743                 $allRelationships = CRM_Core_PseudoConstant::relationshipType('name');
 744                 $relationshipTypeId = str_replace([
 745                   '_a_b',
 746                   '_b_a',
 747                 ], [
 748                   '',
 749                   '',
 750                 ], $key);
 751                 $relationshipType = str_replace($relationshipTypeId . '_', '', $key);
 752                 $orgId = $individualId = NULL;
 753                 if ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employee of') {
 754                   $orgId = $relContactId;
 755                   $individualId = $primaryContactId;
 756                 }
 757                 elseif ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employer of') {
 758                   $orgId = $primaryContactId;
 759                   $individualId = $relContactId;
 760                 }
 761                 if ($orgId && $individualId) {
 762                   $currentEmpParams[$individualId] = $orgId;
 763                   CRM_Contact_BAO_Contact_Utils::setCurrentEmployer($currentEmpParams);
 764                 }
 765               }
 766             }
 767           }
 768         }
 769       }
 770     }
 771     if ($this->_updateWithId) {
 772       //return warning if street address is unparsed, CRM-5886
 773       return $this->processMessage($values, $this->_retCode);
 774     }
 775     //dupe checking
 776     if (is_array($newContact) && civicrm_error($newContact)) {
 777       $code = NULL;
 778
 779       if (($code = CRM_Utils_Array::value('code', $newContact['error_message'])) && ($code == CRM_Core_Error::DUPLICATE_CONTACT)) {
 780         return $this->handleDuplicateError($newContact, $statusFieldName, $values, $onDuplicate, $formatted, $contactFields);
 781       }
 782       // Not a dupe, so we had an error
 783       $errorMessage = $newContact['error_message'];
 784       array_unshift($values, $errorMessage);
 785       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 786       return CRM_Import_Parser::ERROR;
 787
 788     }
 789
 790     if (empty($this->_unparsedStreetAddressContacts)) {
 791       $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '', $contactID);
 792       return CRM_Import_Parser::VALID;
 793     }
 794
 795     // @todo - record unparsed address as 'imported' but the presence of a message is meaningful?
 796     return $this->processMessage($values, CRM_Import_Parser::VALID);
 797   }
 798
 799   /**
 800    * Only called from import now... plus one place outside of core & tests.
 801    *
 802    * @todo - deprecate more aggressively - will involve copying to the import
 803    * class, adding a deprecation notice here & removing from tests.
 804    *
 805    * Takes an associative array and creates a relationship object.
 806    *
 807    * @deprecated For single creates use the api instead (it's tested).
 808    * For multiple a new variant of this function needs to be written and migrated to as this is a bit
 809    * nasty
 810    *
 811    * @param array $params
 812    *   (reference ) an assoc array of name/value pairs.
 813    * @param array $ids
 814    *   The array that holds all the db ids.
 815    *   per http://wiki.civicrm.org/confluence/display/CRM/Database+layer
 816    *  "we are moving away from the $ids param "
 817    *
 818    * @return array
 819    * @throws \CRM_Core_Exception
 820    */
 821   private static function legacyCreateMultiple($params, $ids = []) {
 822     // clarify that the only key ever pass in the ids array is 'contact'
 823     // There is legacy handling for other keys but a universe search on
 824     // calls to this function (not supported to be called from outside core)
 825     // only returns 2 calls - one in CRM_Contact_Import_Parser_Contact
 826     // and the other in jma grant applications (CRM_Grant_Form_Grant_Confirm)
 827     // both only pass in contact as a key here.
 828     $contactID = $ids['contact'];
 829     unset($ids);
 830     // There is only ever one value passed in from the 2 places above that call
 831     // this - by clarifying here like this we can cleanup within this
 832     // function without having to do more universe searches.
 833     $relatedContactID = key($params['contact_check']);
 834
 835     // check if the relationship is valid between contacts.
 836     // step 1: check if the relationship is valid if not valid skip and keep the count
 837     // step 2: check the if two contacts already have a relationship if yes skip and keep the count
 838     // step 3: if valid relationship then add the relation and keep the count
 839
 840     // step 1
 841     [$contactFields['relationship_type_id'], $firstLetter, $secondLetter] = explode('_', $params['relationship_type_id']);
 842     $contactFields['contact_id_' . $firstLetter] = $contactID;
 843     $contactFields['contact_id_' . $secondLetter] = $relatedContactID;
 844     if (!CRM_Contact_BAO_Relationship::checkRelationshipType($contactFields['contact_id_a'], $contactFields['contact_id_b'],
 845       $contactFields['relationship_type_id'])) {
 846       return [0, 0];
 847     }
 848
 849     if (
 850       CRM_Contact_BAO_Relationship::checkDuplicateRelationship(
 851         $contactFields,
 852         $contactID,
 853         // step 2
 854         $relatedContactID
 855       )
 856     ) {
 857       return [0, 1];
 858     }
 859
 860     $singleInstanceParams = array_merge($params, $contactFields);
 861     CRM_Contact_BAO_Relationship::add($singleInstanceParams);
 862     return [1, 0];
 863   }
 864
 865   /**
 866    * Format common params data to proper format to store.
 867    *
 868    * @param array $params
 869    *   Contain record values.
 870    * @param array $formatted
 871    *   Array of formatted data.
 872    * @param array $contactFields
 873    *   Contact DAO fields.
 874    */
 875   private function formatCommonData($params, &$formatted, $contactFields) {
 876     $csType = [
 877       CRM_Utils_Array::value('contact_type', $formatted),
 878     ];
 879
 880     //CRM-5125
 881     //add custom fields for contact sub type
 882     if (!empty($this->_contactSubType)) {
 883       $csType = $this->_contactSubType;
 884     }
 885
 886     if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $formatted)) {
 887       $csType = $relCsType;
 888     }
 889
 890     $customFields = CRM_Core_BAO_CustomField::getFields($formatted['contact_type'], FALSE, FALSE, $csType);
 891
 892     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
 893     $customFields = $customFields + $addressCustomFields;
 894
 895     //if a Custom Email Greeting, Custom Postal Greeting or Custom Addressee is mapped, and no "Greeting / Addressee Type ID" is provided, then automatically set the type = Customized, CRM-4575
 896     $elements = [
 897       'email_greeting_custom' => 'email_greeting',
 898       'postal_greeting_custom' => 'postal_greeting',
 899       'addressee_custom' => 'addressee',
 900     ];
 901     foreach ($elements as $k => $v) {
 902       if (array_key_exists($k, $params) && !(array_key_exists($v, $params))) {
 903         $label = key(CRM_Core_OptionGroup::values($v, TRUE, NULL, NULL, 'AND v.name = "Customized"'));
 904         $params[$v] = $label;
 905       }
 906     }
 907
 908     //format date first
 909     $session = CRM_Core_Session::singleton();
 910     $dateType = $session->get("dateTypes");
 911     foreach ($params as $key => $val) {
 912       $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 913       if ($customFieldID &&
 914         !array_key_exists($customFieldID, $addressCustomFields)
 915       ) {
 916         //we should not update Date to null, CRM-4062
 917         if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
 918           //CRM-21267
 919           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
 920         }
 921         elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
 922           if (empty($val) && !is_numeric($val) && $this->_onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) {
 923             //retain earlier value when Import mode is `Fill`
 924             unset($params[$key]);
 925           }
 926           else {
 927             $params[$key] = CRM_Utils_String::strtoboolstr($val);
 928           }
 929         }
 930       }
 931
 932       if ($key == 'birth_date' && $val) {
 933         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
 934       }
 935       elseif ($key == 'deceased_date' && $val) {
 936         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
 937         $params['is_deceased'] = 1;
 938       }
 939       elseif ($key == 'is_deceased' && $val) {
 940         $params[$key] = CRM_Utils_String::strtoboolstr($val);
 941       }
 942     }
 943
 944     //now format custom data.
 945     foreach ($params as $key => $field) {
 946       if (is_array($field)) {
 947         $isAddressCustomField = FALSE;
 948         foreach ($field as $value) {
 949           $break = FALSE;
 950           if (is_array($value)) {
 951             foreach ($value as $name => $testForEmpty) {
 952               if ($addressCustomFieldID = CRM_Core_BAO_CustomField::getKeyID($name)) {
 953                 $isAddressCustomField = TRUE;
 954                 break;
 955               }
 956               // check if $value does not contain IM provider or phoneType
 957               if (($name !== 'phone_type_id' || $name !== 'provider_id') && ($testForEmpty === '' || $testForEmpty == NULL)) {
 958                 $break = TRUE;
 959                 break;
 960               }
 961             }
 962           }
 963           else {
 964             $break = TRUE;
 965           }
 966
 967           if (!$break) {
 968             if (!empty($value['location_type_id'])) {
 969               $this->formatLocationBlock($value, $formatted);
 970             }
 971             else {
 972               // @todo - this is still reachable - e.g. import with related contact info like firstname,lastname,spouse-first-name,spouse-last-name,spouse-home-phone
 973               CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
 974               $this->formatContactParameters($value, $formatted);
 975             }
 976           }
 977         }
 978         if (!$isAddressCustomField) {
 979           continue;
 980         }
 981       }
 982
 983       $formatValues = [
 984         $key => $field,
 985       ];
 986
 987       if (($key !== 'preferred_communication_method') && (array_key_exists($key, $contactFields))) {
 988         // due to merging of individual table and
 989         // contact table, we need to avoid
 990         // preferred_communication_method forcefully
 991         $formatValues['contact_type'] = $formatted['contact_type'];
 992       }
 993
 994       if ($key == 'id' && isset($field)) {
 995         $formatted[$key] = $field;
 996       }
 997       $this->formatContactParameters($formatValues, $formatted);
 998
 999       //Handling Custom Data
1000       // note: Address custom fields will be handled separately inside formatContactParameters
1001       if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
1002         array_key_exists($customFieldID, $customFields) &&
1003         !array_key_exists($customFieldID, $addressCustomFields)
1004       ) {
1005
1006         $extends = $customFields[$customFieldID]['extends'] ?? NULL;
1007         $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1008         $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
1009         $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1010
1011         if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
1012           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1013           foreach ($customOption as $customValue) {
1014             $val = $customValue['value'] ?? NULL;
1015             $label = strtolower($customValue['label'] ?? '');
1016             $value = strtolower(trim($formatted[$key]));
1017             if (($value == $label) || ($value == strtolower($val))) {
1018               $params[$key] = $formatted[$key] = $val;
1019             }
1020           }
1021         }
1022         elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
1023           $mulValues = explode(',', $formatted[$key]);
1024           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1025           $formatted[$key] = [];
1026           $params[$key] = [];
1027           foreach ($mulValues as $v1) {
1028             foreach ($customOption as $v2) {
1029               if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1030                 (strtolower($v2['value']) == strtolower(trim($v1)))
1031               ) {
1032                 if ($htmlType == 'CheckBox') {
1033                   $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
1034                 }
1035                 else {
1036                   $params[$key][] = $formatted[$key][] = $v2['value'];
1037                 }
1038               }
1039             }
1040           }
1041         }
1042       }
1043     }
1044
1045     if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields) &&
1046       !array_key_exists($customFieldID, $addressCustomFields)
1047     ) {
1048       // @todo calling api functions directly is not supported
1049       _civicrm_api3_custom_format_params($params, $formatted, $extends);
1050     }
1051
1052     // to check if not update mode and unset the fields with empty value.
1053     if (!$this->_updateWithId && array_key_exists('custom', $formatted)) {
1054       foreach ($formatted['custom'] as $customKey => $customvalue) {
1055         if (empty($formatted['custom'][$customKey][-1]['is_required'])) {
1056           $formatted['custom'][$customKey][-1]['is_required'] = $customFields[$customKey]['is_required'];
1057         }
1058         $emptyValue = $customvalue[-1]['value'] ?? NULL;
1059         if (!isset($emptyValue)) {
1060           unset($formatted['custom'][$customKey]);
1061         }
1062       }
1063     }
1064
1065     // parse street address, CRM-5450
1066     if ($this->_parseStreetAddress) {
1067       if (array_key_exists('address', $formatted) && is_array($formatted['address'])) {
1068         foreach ($formatted['address'] as $instance => & $address) {
1069           $streetAddress = $address['street_address'] ?? NULL;
1070           if (empty($streetAddress)) {
1071             continue;
1072           }
1073           // parse address field.
1074           $parsedFields = CRM_Core_BAO_Address::parseStreetAddress($streetAddress);
1075
1076           //street address consider to be parsed properly,
1077           //If we get street_name and street_number.
1078           if (empty($parsedFields['street_name']) || empty($parsedFields['street_number'])) {
1079             $parsedFields = array_fill_keys(array_keys($parsedFields), '');
1080           }
1081
1082           // merge parse address w/ main address block.
1083           $address = array_merge($address, $parsedFields);
1084         }
1085       }
1086     }
1087   }
1088
1089   /**
1090    * Get the array of successfully imported contact id's
1091    *
1092    * @return array
1093    */
1094   public function getImportedContacts() {
1095     return $this->_newContacts;
1096   }
1097
1098   /**
1099    * Get the array of successfully imported related contact id's
1100    *
1101    * @return array
1102    */
1103   public function &getRelatedImportedContacts() {
1104     return $this->_newRelatedContacts;
1105   }
1106
1107   /**
1108    * Check if an error in custom data.
1109    *
1110    * @param array $params
1111    * @param string $errorMessage
1112    *   A string containing all the error-fields.
1113    *
1114    * @param null $csType
1115    * @param null $relationships
1116    */
1117   public static function isErrorInCustomData($params, &$errorMessage, $csType = NULL, $relationships = NULL) {
1118     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1119     $errors = [];
1120
1121     if (!empty($params['contact_sub_type'])) {
1122       $csType = $params['contact_sub_type'] ?? NULL;
1123     }
1124
1125     if (empty($params['contact_type'])) {
1126       $params['contact_type'] = 'Individual';
1127     }
1128
1129     // get array of subtypes - CRM-18708
1130     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1131       $csType = self::getSubtypes($params['contact_type']);
1132     }
1133
1134     if (is_array($csType)) {
1135       // fetch custom fields for every subtype and add it to $customFields array
1136       // CRM-18708
1137       $customFields = [];
1138       foreach ($csType as $cType) {
1139         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1140       }
1141     }
1142     else {
1143       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1144     }
1145
1146     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
1147     $customFields = $customFields + $addressCustomFields;
1148     foreach ($params as $key => $value) {
1149       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1150         /* check if it's a valid custom field id */
1151
1152         if (!array_key_exists($customFieldID, $customFields)) {
1153           $errors[] = ts('field ID');
1154         }
1155         // validate null values for required custom fields of type boolean
1156         if (!empty($customFields[$customFieldID]['is_required']) && (empty($params['custom_' . $customFieldID]) && !is_numeric($params['custom_' . $customFieldID])) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
1157           $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1158         }
1159
1160         //For address custom fields, we do get actual custom field value as an inner array of
1161         //values so need to modify
1162         if (array_key_exists($customFieldID, $addressCustomFields)) {
1163           $value = $value[0][$key];
1164         }
1165         /* validate the data against the CF type */
1166
1167         if ($value) {
1168           $dataType = $customFields[$customFieldID]['data_type'];
1169           $htmlType = $customFields[$customFieldID]['html_type'];
1170           $isSerialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1171           if ($dataType == 'Date') {
1172             if (array_key_exists($customFieldID, $addressCustomFields) && CRM_Utils_Date::convertToDefaultDate($params[$key][0], $dateType, $key)) {
1173               $value = $params[$key][0][$key];
1174             }
1175             elseif (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1176               $value = $params[$key];
1177             }
1178             else {
1179               $errors[] = $customFields[$customFieldID]['label'];
1180             }
1181           }
1182           elseif ($dataType == 'Boolean') {
1183             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1184               $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1185             }
1186           }
1187           // need not check for label filed import
1188           $selectHtmlTypes = [
1189             'CheckBox',
1190             'Select',
1191             'Radio',
1192           ];
1193           if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1194             $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1195             if (!$valid) {
1196               $errors[] = $customFields[$customFieldID]['label'];
1197             }
1198           }
1199
1200           // check for values for custom fields for checkboxes and multiselect
1201           if ($isSerialized && $dataType != 'ContactReference') {
1202             $value = trim($value);
1203             $value = str_replace('|', ',', $value);
1204             $mulValues = explode(',', $value);
1205             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1206             foreach ($mulValues as $v1) {
1207               if (strlen($v1) == 0) {
1208                 continue;
1209               }
1210
1211               $flag = FALSE;
1212               foreach ($customOption as $v2) {
1213                 if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1214                   $flag = TRUE;
1215                 }
1216               }
1217
1218               if (!$flag) {
1219                 $errors[] = $customFields[$customFieldID]['label'];
1220               }
1221             }
1222           }
1223           elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1224             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1225             $flag = FALSE;
1226             foreach ($customOption as $v2) {
1227               if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1228                 $flag = TRUE;
1229               }
1230             }
1231             if (!$flag) {
1232               $errors[] = $customFields[$customFieldID]['label'];
1233             }
1234           }
1235           elseif ($isSerialized && $dataType === 'StateProvince') {
1236             $mulValues = explode(',', $value);
1237             foreach ($mulValues as $stateValue) {
1238               if ($stateValue) {
1239                 if (self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvinceAbbreviation()) || self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvince())) {
1240                   continue;
1241                 }
1242                 else {
1243                   $errors[] = $customFields[$customFieldID]['label'];
1244                 }
1245               }
1246             }
1247           }
1248           elseif ($isSerialized && $dataType == 'Country') {
1249             $mulValues = explode(',', $value);
1250             foreach ($mulValues as $countryValue) {
1251               if ($countryValue) {
1252                 CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1253                 CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1254                 $limitCodes = CRM_Core_BAO_Country::countryLimit();
1255
1256                 $error = TRUE;
1257                 foreach ([
1258                   $countryNames,
1259                   $countryIsoCodes,
1260                   $limitCodes,
1261                 ] as $values) {
1262                   if (in_array(trim($countryValue), $values)) {
1263                     $error = FALSE;
1264                     break;
1265                   }
1266                 }
1267
1268                 if ($error) {
1269                   $errors[] = $customFields[$customFieldID]['label'];
1270                 }
1271               }
1272             }
1273           }
1274         }
1275       }
1276       elseif (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1277         //CRM-5125
1278         //supporting custom data of related contact subtypes
1279         $relation = NULL;
1280         if ($relationships) {
1281           if (array_key_exists($key, $relationships)) {
1282             $relation = $key;
1283           }
1284           elseif (CRM_Utils_Array::key($key, $relationships)) {
1285             $relation = CRM_Utils_Array::key($key, $relationships);
1286           }
1287         }
1288         if (!empty($relation)) {
1289           [$id, $first, $second] = CRM_Utils_System::explode('_', $relation, 3);
1290           $direction = "contact_sub_type_$second";
1291           $relationshipType = new CRM_Contact_BAO_RelationshipType();
1292           $relationshipType->id = $id;
1293           if ($relationshipType->find(TRUE)) {
1294             if (isset($relationshipType->$direction)) {
1295               $params[$key]['contact_sub_type'] = $relationshipType->$direction;
1296             }
1297           }
1298         }
1299
1300         self::isErrorInCustomData($params[$key], $errorMessage, $csType, $relationships);
1301       }
1302     }
1303     if ($errors) {
1304       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1305     }
1306   }
1307
1308   /**
1309    * Check if value present in all genders or.
1310    * as a substring of any gender value, if yes than return corresponding gender.
1311    * eg value might be  m/M, ma/MA, mal/MAL, male return 'Male'
1312    * but if value is 'maleabc' than return false
1313    *
1314    * @param string $gender
1315    *   Check this value across gender values.
1316    *
1317    * retunr gender value / false
1318    *
1319    * @return bool
1320    */
1321   public function checkGender($gender) {
1322     $gender = trim($gender, '.');
1323     if (!$gender) {
1324       return FALSE;
1325     }
1326
1327     $allGenders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
1328     foreach ($allGenders as $key => $value) {
1329       if (strlen($gender) > strlen($value)) {
1330         continue;
1331       }
1332       if ($gender == $value) {
1333         return $value;
1334       }
1335       if (substr_compare($value, $gender, 0, strlen($gender), TRUE) === 0) {
1336         return $value;
1337       }
1338     }
1339
1340     return FALSE;
1341   }
1342
1343   /**
1344    * Check if an error in Core( non-custom fields ) field
1345    *
1346    * @param array $params
1347    * @param string $errorMessage
1348    *   A string containing all the error-fields.
1349    */
1350   public function isErrorInCoreData($params, &$errorMessage) {
1351     $errors = [];
1352     foreach ($params as $key => $value) {
1353       if ($value) {
1354         $session = CRM_Core_Session::singleton();
1355         $dateType = $session->get("dateTypes");
1356
1357         switch ($key) {
1358           case 'birth_date':
1359             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1360               if (!CRM_Utils_Rule::date($params[$key])) {
1361                 $errors[] = ts('Birth Date');
1362               }
1363             }
1364             else {
1365               $errors[] = ts('Birth-Date');
1366             }
1367             break;
1368
1369           case 'deceased_date':
1370             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1371               if (!CRM_Utils_Rule::date($params[$key])) {
1372                 $errors[] = ts('Deceased Date');
1373               }
1374             }
1375             else {
1376               $errors[] = ts('Deceased Date');
1377             }
1378             break;
1379
1380           case 'is_deceased':
1381             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1382               $errors[] = ts('Deceased');
1383             }
1384             break;
1385
1386           case 'gender_id':
1387             if (!self::checkGender($value)) {
1388               $errors[] = ts('Gender');
1389             }
1390             break;
1391
1392           case 'preferred_communication_method':
1393             $preffComm = [];
1394             $preffComm = explode(',', $value);
1395             foreach ($preffComm as $v) {
1396               if (!self::in_value(trim($v), CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method'))) {
1397                 $errors[] = ts('Preferred Communication Method');
1398               }
1399             }
1400             break;
1401
1402           case 'preferred_mail_format':
1403             if (!array_key_exists(strtolower($value), array_change_key_case(CRM_Core_SelectValues::pmf(), CASE_LOWER))) {
1404               $errors[] = ts('Preferred Mail Format');
1405             }
1406             break;
1407
1408           case 'individual_prefix':
1409           case 'prefix_id':
1410             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id'))) {
1411               $errors[] = ts('Individual Prefix');
1412             }
1413             break;
1414
1415           case 'individual_suffix':
1416           case 'suffix_id':
1417             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id'))) {
1418               $errors[] = ts('Individual Suffix');
1419             }
1420             break;
1421
1422           case 'state_province':
1423             if (!empty($value)) {
1424               foreach ($value as $stateValue) {
1425                 if ($stateValue['state_province']) {
1426                   if (self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvinceAbbreviation()) ||
1427                     self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvince())
1428                   ) {
1429                     continue;
1430                   }
1431                   else {
1432                     $errors[] = ts('State/Province');
1433                   }
1434                 }
1435               }
1436             }
1437             break;
1438
1439           case 'country':
1440             if (!empty($value)) {
1441               foreach ($value as $stateValue) {
1442                 if ($stateValue['country']) {
1443                   CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1444                   CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1445                   $limitCodes = CRM_Core_BAO_Country::countryLimit();
1446                   //If no country is selected in
1447                   //localization then take all countries
1448                   if (empty($limitCodes)) {
1449                     $limitCodes = $countryIsoCodes;
1450                   }
1451
1452                   if (self::in_value($stateValue['country'], $limitCodes) || self::in_value($stateValue['country'], CRM_Core_PseudoConstant::country())) {
1453                     continue;
1454                   }
1455                   if (self::in_value($stateValue['country'], $countryIsoCodes) || self::in_value($stateValue['country'], $countryNames)) {
1456                     $errors[] = ts('Country input value is in table but not "available": "This Country is valid but is NOT in the list of Available Countries currently configured for your site. This can be viewed and modifed from Administer > Localization > Languages Currency Locations." ');
1457                   }
1458                   else {
1459                     $errors[] = ts('Country input value not in country table: "The Country value appears to be invalid. It does not match any value in CiviCRM table of countries."');
1460                   }
1461                 }
1462               }
1463             }
1464             break;
1465
1466           case 'county':
1467             if (!empty($value)) {
1468               foreach ($value as $county) {
1469                 if ($county['county']) {
1470                   $countyNames = CRM_Core_PseudoConstant::county();
1471                   if (!empty($county['county']) && !in_array($county['county'], $countyNames)) {
1472                     $errors[] = ts('County input value not in county table: The County value appears to be invalid. It does not match any value in CiviCRM table of counties.');
1473                   }
1474                 }
1475               }
1476             }
1477             break;
1478
1479           case 'geo_code_1':
1480             if (!empty($value)) {
1481               foreach ($value as $codeValue) {
1482                 if (!empty($codeValue['geo_code_1'])) {
1483                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_1'])) {
1484                     continue;
1485                   }
1486                   $errors[] = ts('Geo code 1');
1487                 }
1488               }
1489             }
1490             break;
1491
1492           case 'geo_code_2':
1493             if (!empty($value)) {
1494               foreach ($value as $codeValue) {
1495                 if (!empty($codeValue['geo_code_2'])) {
1496                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_2'])) {
1497                     continue;
1498                   }
1499                   $errors[] = ts('Geo code 2');
1500                 }
1501               }
1502             }
1503             break;
1504
1505           //check for any error in email/postal greeting, addressee,
1506           //custom email/postal greeting, custom addressee, CRM-4575
1507
1508           case 'email_greeting':
1509             $emailGreetingFilter = [
1510               'contact_type' => $this->_contactType,
1511               'greeting_type' => 'email_greeting',
1512             ];
1513             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($emailGreetingFilter))) {
1514               $errors[] = ts('Email Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Email Greetings for valid values');
1515             }
1516             break;
1517
1518           case 'postal_greeting':
1519             $postalGreetingFilter = [
1520               'contact_type' => $this->_contactType,
1521               'greeting_type' => 'postal_greeting',
1522             ];
1523             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($postalGreetingFilter))) {
1524               $errors[] = ts('Postal Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Postal Greetings for valid values');
1525             }
1526             break;
1527
1528           case 'addressee':
1529             $addresseeFilter = [
1530               'contact_type' => $this->_contactType,
1531               'greeting_type' => 'addressee',
1532             ];
1533             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($addresseeFilter))) {
1534               $errors[] = ts('Addressee must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Addressee for valid values');
1535             }
1536             break;
1537
1538           case 'email_greeting_custom':
1539             if (array_key_exists('email_greeting', $params)) {
1540               $emailGreetingLabel = key(CRM_Core_OptionGroup::values('email_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1541               if (CRM_Utils_Array::value('email_greeting', $params) != $emailGreetingLabel) {
1542                 $errors[] = ts('Email Greeting - Custom');
1543               }
1544             }
1545             break;
1546
1547           case 'postal_greeting_custom':
1548             if (array_key_exists('postal_greeting', $params)) {
1549               $postalGreetingLabel = key(CRM_Core_OptionGroup::values('postal_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1550               if (CRM_Utils_Array::value('postal_greeting', $params) != $postalGreetingLabel) {
1551                 $errors[] = ts('Postal Greeting - Custom');
1552               }
1553             }
1554             break;
1555
1556           case 'addressee_custom':
1557             if (array_key_exists('addressee', $params)) {
1558               $addresseeLabel = key(CRM_Core_OptionGroup::values('addressee', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1559               if (CRM_Utils_Array::value('addressee', $params) != $addresseeLabel) {
1560                 $errors[] = ts('Addressee - Custom');
1561               }
1562             }
1563             break;
1564
1565           case 'url':
1566             if (is_array($value)) {
1567               foreach ($value as $values) {
1568                 if (!empty($values['url']) && !CRM_Utils_Rule::url($values['url'])) {
1569                   $errors[] = ts('Website');
1570                   break;
1571                 }
1572               }
1573             }
1574             break;
1575
1576           case 'do_not_email':
1577           case 'do_not_phone':
1578           case 'do_not_mail':
1579           case 'do_not_sms':
1580           case 'do_not_trade':
1581             if (CRM_Utils_Rule::boolean($value) == FALSE) {
1582               $key = ucwords(str_replace("_", " ", $key));
1583               $errors[] = $key;
1584             }
1585             break;
1586
1587           case 'email':
1588             if (is_array($value)) {
1589               foreach ($value as $values) {
1590                 if (!empty($values['email']) && !CRM_Utils_Rule::email($values['email'])) {
1591                   $errors[] = $key;
1592                   break;
1593                 }
1594               }
1595             }
1596             break;
1597
1598           default:
1599             if (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1600               //check for any relationship data ,FIX ME
1601               self::isErrorInCoreData($params[$key], $errorMessage);
1602             }
1603         }
1604       }
1605     }
1606     if ($errors) {
1607       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1608     }
1609   }
1610
1611   /**
1612    * Ckeck a value present or not in a array.
1613    *
1614    * @param $value
1615    * @param $valueArray
1616    *
1617    * @return bool
1618    */
1619   public static function in_value($value, $valueArray) {
1620     foreach ($valueArray as $key => $v) {
1621       //fix for CRM-1514
1622       if (strtolower(trim($v, ".")) == strtolower(trim($value, "."))) {
1623         return TRUE;
1624       }
1625     }
1626     return FALSE;
1627   }
1628
1629   /**
1630    * Build error-message containing error-fields
1631    *
1632    * Once upon a time there was a dev who hadn't heard of implode. That dev wrote this function.
1633    *
1634    * @todo just say no!
1635    *
1636    * @param string $errorName
1637    *   A string containing error-field name.
1638    * @param string $errorMessage
1639    *   A string containing all the error-fields, where the new errorName is concatenated.
1640    *
1641    */
1642   public static function addToErrorMsg($errorName, &$errorMessage) {
1643     if ($errorMessage) {
1644       $errorMessage .= "; $errorName";
1645     }
1646     else {
1647       $errorMessage = $errorName;
1648     }
1649   }
1650
1651   /**
1652    * Method for creating contact.
1653    *
1654    * @param array $formatted
1655    * @param array $contactFields
1656    * @param int $onDuplicate
1657    * @param int $contactId
1658    * @param bool $requiredCheck
1659    * @param int $dedupeRuleGroupID
1660    *
1661    * @return array|bool|\CRM_Contact_BAO_Contact|\CRM_Core_Error|null
1662    */
1663   public function createContact(&$formatted, &$contactFields, $onDuplicate, $contactId = NULL, $requiredCheck = TRUE, $dedupeRuleGroupID = NULL) {
1664     $dupeCheck = FALSE;
1665     $newContact = NULL;
1666
1667     if (is_null($contactId) && ($onDuplicate != CRM_Import_Parser::DUPLICATE_NOCHECK)) {
1668       $dupeCheck = (bool) ($onDuplicate);
1669     }
1670
1671     //get the prefix id etc if exists
1672     CRM_Contact_BAO_Contact::resolveDefaults($formatted, TRUE);
1673
1674     //@todo direct call to API function not supported.
1675     // setting required check to false, CRM-2839
1676     // plus we do our own required check in import
1677     try {
1678       $error = $this->deprecated_contact_check_params($formatted, $dupeCheck, $dedupeRuleGroupID);
1679       if ($error) {
1680         return $error;
1681       }
1682       $this->deprecated_validate_formatted_contact($formatted);
1683     }
1684     catch (CRM_Core_Exception $e) {
1685       return ['error_message' => $e->getMessage(), 'is_error' => 1, 'code' => $e->getCode()];
1686     }
1687
1688     if ($contactId) {
1689       $this->formatParams($formatted, $onDuplicate, (int) $contactId);
1690     }
1691
1692     // Resetting and rebuilding cache could be expensive.
1693     CRM_Core_Config::setPermitCacheFlushMode(FALSE);
1694
1695     // If a user has logged in, or accessed via a checksum
1696     // Then deliberately 'blanking' a value in the profile should remove it from their record
1697     // @todo this should either be TRUE or FALSE in the context of import - once
1698     // we figure out which we can remove all the rest.
1699     // Also note the meaning of this parameter is less than it used to
1700     // be following block cleanup.
1701     $formatted['updateBlankLocInfo'] = TRUE;
1702     if ((CRM_Core_Session::singleton()->get('authSrc') & (CRM_Core_Permission::AUTH_SRC_CHECKSUM + CRM_Core_Permission::AUTH_SRC_LOGIN)) == 0) {
1703       $formatted['updateBlankLocInfo'] = FALSE;
1704     }
1705
1706     [$data, $contactDetails] = CRM_Contact_BAO_Contact::formatProfileContactParams($formatted, $contactFields, $contactId, NULL, $formatted['contact_type']);
1707
1708     // manage is_opt_out
1709     if (array_key_exists('is_opt_out', $contactFields) && array_key_exists('is_opt_out', $formatted)) {
1710       $wasOptOut = $contactDetails['is_opt_out'] ?? FALSE;
1711       $isOptOut = $formatted['is_opt_out'];
1712       $data['is_opt_out'] = $isOptOut;
1713       // on change, create new civicrm_subscription_history entry
1714       if (($wasOptOut != $isOptOut) && !empty($contactDetails['contact_id'])) {
1715         $shParams = [
1716           'contact_id' => $contactDetails['contact_id'],
1717           'status' => $isOptOut ? 'Removed' : 'Added',
1718           'method' => 'Web',
1719         ];
1720         CRM_Contact_BAO_SubscriptionHistory::create($shParams);
1721       }
1722     }
1723
1724     $contact = civicrm_api3('Contact', 'create', $data);
1725     $cid = $contact['id'];
1726
1727     CRM_Core_Config::setPermitCacheFlushMode(TRUE);
1728
1729     $contact = [
1730       'contact_id' => $cid,
1731     ];
1732
1733     $defaults = [];
1734     $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
1735
1736     //get the id of the contact whose street address is not parsable, CRM-5886
1737     if ($this->_parseStreetAddress && is_object($newContact) && property_exists($newContact, 'address') && $newContact->address) {
1738       foreach ($newContact->address as $address) {
1739         if (!empty($address['street_address']) && (empty($address['street_number']) || empty($address['street_name']))) {
1740           $this->_unparsedStreetAddressContacts[] = [
1741             'id' => $newContact->id,
1742             'streetAddress' => $address['street_address'],
1743           ];
1744         }
1745       }
1746     }
1747     return $newContact;
1748   }
1749
1750   /**
1751    * Format params for update and fill mode.
1752    *
1753    * @param array $params
1754    *   reference to an array containing all the.
1755    *   values for import
1756    * @param int $onDuplicate
1757    * @param int $cid
1758    *   contact id.
1759    */
1760   public function formatParams(&$params, $onDuplicate, $cid) {
1761     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
1762       return;
1763     }
1764
1765     $contactParams = [
1766       'contact_id' => $cid,
1767     ];
1768
1769     $defaults = [];
1770     $contactObj = CRM_Contact_BAO_Contact::retrieve($contactParams, $defaults);
1771
1772     $modeFill = ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL);
1773
1774     $groupTree = CRM_Core_BAO_CustomGroup::getTree($params['contact_type'], NULL, $cid, 0, NULL);
1775     CRM_Core_BAO_CustomGroup::setDefaults($groupTree, $defaults, FALSE, FALSE);
1776
1777     $locationFields = [
1778       'email' => 'email',
1779       'phone' => 'phone',
1780       'im' => 'name',
1781       'website' => 'website',
1782       'address' => 'address',
1783     ];
1784
1785     $contact = get_object_vars($contactObj);
1786
1787     foreach ($params as $key => $value) {
1788       if ($key == 'id' || $key == 'contact_type') {
1789         continue;
1790       }
1791
1792       if (array_key_exists($key, $locationFields)) {
1793         continue;
1794       }
1795       if (in_array($key, [
1796         'email_greeting',
1797         'postal_greeting',
1798         'addressee',
1799       ])) {
1800         // CRM-4575, need to null custom
1801         if ($params["{$key}_id"] != 4) {
1802           $params["{$key}_custom"] = 'null';
1803         }
1804         unset($params[$key]);
1805       }
1806       else {
1807         if ($customFieldId = CRM_Core_BAO_CustomField::getKeyID($key)) {
1808           $custom_params = ['id' => $contact['id'], 'return' => $key];
1809           $getValue = civicrm_api3('Contact', 'getvalue', $custom_params);
1810           if (empty($getValue)) {
1811             unset($getValue);
1812           }
1813         }
1814         else {
1815           $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $key);
1816         }
1817         if ($key == 'contact_source') {
1818           $params['source'] = $params[$key];
1819           unset($params[$key]);
1820         }
1821
1822         if ($modeFill && isset($getValue)) {
1823           unset($params[$key]);
1824           if ($customFieldId) {
1825             // Extra values must be unset to ensure the values are not
1826             // imported.
1827             unset($params['custom'][$customFieldId]);
1828           }
1829         }
1830       }
1831     }
1832
1833     foreach ($locationFields as $locKeys) {
1834       if (isset($params[$locKeys]) && is_array($params[$locKeys])) {
1835         foreach ($params[$locKeys] as $key => $value) {
1836           if ($modeFill) {
1837             $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $locKeys);
1838
1839             if (isset($getValue)) {
1840               foreach ($getValue as $cnt => $values) {
1841                 if ($locKeys == 'website') {
1842                   if (($getValue[$cnt]['website_type_id'] == $params[$locKeys][$key]['website_type_id'])) {
1843                     unset($params[$locKeys][$key]);
1844                   }
1845                 }
1846                 else {
1847                   if ((!empty($getValue[$cnt]['location_type_id']) && !empty($params[$locKeys][$key]['location_type_id'])) && $getValue[$cnt]['location_type_id'] == $params[$locKeys][$key]['location_type_id']) {
1848                     unset($params[$locKeys][$key]);
1849                   }
1850                 }
1851               }
1852             }
1853           }
1854         }
1855         if (count($params[$locKeys]) == 0) {
1856           unset($params[$locKeys]);
1857         }
1858       }
1859     }
1860   }
1861
1862   /**
1863    * Convert any given date string to default date array.
1864    *
1865    * @param array $params
1866    *   Has given date-format.
1867    * @param array $formatted
1868    *   Store formatted date in this array.
1869    * @param int $dateType
1870    *   Type of date.
1871    * @param string $dateParam
1872    *   Index of params.
1873    */
1874   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
1875     //fix for CRM-2687
1876     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
1877     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
1878   }
1879
1880   /**
1881    * Generate status and error message for unparsed street address records.
1882    *
1883    * @param array $values
1884    *   The array of values belonging to each row.
1885    * @param $returnCode
1886    *
1887    * @return int
1888    */
1889   private function processMessage(&$values, $returnCode) {
1890     if (empty($this->_unparsedStreetAddressContacts)) {
1891       $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '');
1892     }
1893     else {
1894       $errorMessage = ts("Record imported successfully but unable to parse the street address: ");
1895       foreach ($this->_unparsedStreetAddressContacts as $contactInfo => $contactValue) {
1896         $contactUrl = CRM_Utils_System::url('civicrm/contact/add', 'reset=1&action=update&cid=' . $contactValue['id'], TRUE, NULL, FALSE);
1897         $errorMessage .= "\n Contact ID:" . $contactValue['id'] . " <a href=\"$contactUrl\"> " . $contactValue['streetAddress'] . "</a>";
1898       }
1899       array_unshift($values, $errorMessage);
1900       $returnCode = CRM_Import_Parser::UNPARSED_ADDRESS_WARNING;
1901       $this->setImportStatus((int) ($values[count($values) - 1]), 'ERROR', $errorMessage);
1902     }
1903     return $returnCode;
1904   }
1905
1906   /**
1907    * get subtypes given the contact type
1908    *
1909    * @param string $contactType
1910    * @return array $subTypes
1911    */
1912   public static function getSubtypes($contactType) {
1913     $subTypes = [];
1914     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
1915
1916     if (count($types) > 0) {
1917       foreach ($types as $type) {
1918         $subTypes[] = $type['name'];
1919       }
1920     }
1921     return $subTypes;
1922   }
1923
1924   /**
1925    * Get the possible contact matches.
1926    *
1927    * 1) the chosen dedupe rule falling back to
1928    * 2) a check for the external ID.
1929    *
1930    * @see https://issues.civicrm.org/jira/browse/CRM-17275
1931    *
1932    * @param array $params
1933    *
1934    * @return array
1935    *   IDs of possible matches.
1936    *
1937    * @throws \CRM_Core_Exception
1938    * @throws \CiviCRM_API3_Exception
1939    */
1940   protected function getPossibleContactMatches($params) {
1941     $extIDMatch = NULL;
1942
1943     if (!empty($params['external_identifier'])) {
1944       // Check for any match on external id, deleted or otherwise.
1945       $extIDContact = civicrm_api3('Contact', 'get', [
1946         'external_identifier' => $params['external_identifier'],
1947         'showAll' => 'all',
1948         'return' => ['id', 'contact_is_deleted'],
1949       ]);
1950       if (isset($extIDContact['id'])) {
1951         $extIDMatch = $extIDContact['id'];
1952
1953         if ($extIDContact['values'][$extIDMatch]['contact_is_deleted'] == 1) {
1954           // If the contact is deleted, update external identifier to be blank
1955           // to avoid key error from MySQL.
1956           $params = ['id' => $extIDMatch, 'external_identifier' => ''];
1957           civicrm_api3('Contact', 'create', $params);
1958
1959           // And now it is no longer a match.
1960           $extIDMatch = NULL;
1961         }
1962       }
1963     }
1964     $checkParams = ['check_permissions' => FALSE, 'match' => $params];
1965     $checkParams['match']['contact_type'] = $this->_contactType;
1966
1967     $possibleMatches = civicrm_api3('Contact', 'duplicatecheck', $checkParams);
1968     if (!$extIDMatch) {
1969       return array_keys($possibleMatches['values']);
1970     }
1971     if ($possibleMatches['count']) {
1972       if (array_key_exists($extIDMatch, $possibleMatches['values'])) {
1973         return [$extIDMatch];
1974       }
1975       throw new CRM_Core_Exception(ts(
1976         'Matching this contact based on the de-dupe rule would cause an external ID conflict'));
1977     }
1978     return [$extIDMatch];
1979   }
1980
1981   /**
1982    * Format the form mapping parameters ready for the parser.
1983    *
1984    * @param int $count
1985    *   Number of rows.
1986    *
1987    * @return array $parserParameters
1988    */
1989   public static function getParameterForParser($count) {
1990     $baseArray = [];
1991     for ($i = 0; $i < $count; $i++) {
1992       $baseArray[$i] = NULL;
1993     }
1994     $parserParameters['mapperLocType'] = $baseArray;
1995     $parserParameters['mapperPhoneType'] = $baseArray;
1996     $parserParameters['mapperImProvider'] = $baseArray;
1997     $parserParameters['mapperWebsiteType'] = $baseArray;
1998     $parserParameters['mapperRelated'] = $baseArray;
1999     $parserParameters['relatedContactType'] = $baseArray;
2000     $parserParameters['relatedContactDetails'] = $baseArray;
2001     $parserParameters['relatedContactLocType'] = $baseArray;
2002     $parserParameters['relatedContactPhoneType'] = $baseArray;
2003     $parserParameters['relatedContactImProvider'] = $baseArray;
2004     $parserParameters['relatedContactWebsiteType'] = $baseArray;
2005
2006     return $parserParameters;
2007
2008   }
2009
2010   /**
2011    * Set field metadata.
2012    */
2013   protected function setFieldMetadata() {
2014     $this->setImportableFieldsMetadata($this->getContactImportMetadata());
2015     // Probably no longer needed but here for now.
2016     $this->_relationships = $this->getRelationships();
2017   }
2018
2019   /**
2020    * @param array $newContact
2021    * @param $statusFieldName
2022    * @param array $values
2023    * @param int $onDuplicate
2024    * @param array $formatted
2025    * @param array $contactFields
2026    *
2027    * @return int
2028    *
2029    * @throws \CRM_Core_Exception
2030    * @throws \CiviCRM_API3_Exception
2031    * @throws \Civi\API\Exception\UnauthorizedException
2032    */
2033   protected function handleDuplicateError(array $newContact, $statusFieldName, array $values, int $onDuplicate, array $formatted, array $contactFields): int {
2034     $urls = [];
2035     // need to fix at some stage and decide if the error will return an
2036     // array or string, crude hack for now
2037     if (is_array($newContact['error_message']['params'][0])) {
2038       $cids = $newContact['error_message']['params'][0];
2039     }
2040     else {
2041       $cids = explode(',', $newContact['error_message']['params'][0]);
2042     }
2043
2044     foreach ($cids as $cid) {
2045       $urls[] = CRM_Utils_System::url('civicrm/contact/view', 'reset=1&cid=' . $cid, TRUE);
2046     }
2047
2048     $url_string = implode("\n", $urls);
2049
2050     // If we duplicate more than one record, skip no matter what
2051     if (count($cids) > 1) {
2052       $errorMessage = ts('Record duplicates multiple contacts');
2053       //combine error msg to avoid mismatch between error file columns.
2054       $errorMessage .= "\n" . $url_string;
2055       array_unshift($values, $errorMessage);
2056       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2057       return CRM_Import_Parser::ERROR;
2058     }
2059
2060     // Params only had one id, so shift it out
2061     $contactId = array_shift($cids);
2062     $cid = NULL;
2063
2064     $vals = ['contact_id' => $contactId];
2065
2066     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_REPLACE) {
2067       civicrm_api('contact', 'delete', $vals);
2068       $cid = CRM_Contact_BAO_Contact::createProfileContact($formatted, $contactFields, $contactId, NULL, NULL, $formatted['contact_type']);
2069     }
2070     if (in_array((int) $onDuplicate, [CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::DUPLICATE_FILL], TRUE)) {
2071       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactId);
2072     }
2073     // else skip does nothing and just returns an error code.
2074     if ($cid) {
2075       $contact = [
2076         'contact_id' => $cid,
2077       ];
2078       $defaults = [];
2079       $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
2080     }
2081
2082     if (civicrm_error($newContact)) {
2083       if (empty($newContact['error_message']['params'])) {
2084         // different kind of error other than DUPLICATE
2085         $errorMessage = $newContact['error_message'];
2086         array_unshift($values, $errorMessage);
2087         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2088         return CRM_Import_Parser::ERROR;
2089       }
2090
2091       $contactID = $newContact['error_message']['params'][0];
2092       if (is_array($contactID)) {
2093         $contactID = array_pop($contactID);
2094       }
2095       if (!in_array($contactID, $this->_newContacts)) {
2096         $this->_newContacts[] = $contactID;
2097       }
2098     }
2099     //CRM-262 No Duplicate Checking
2100     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
2101       array_unshift($values, $url_string);
2102       $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', 'Skipping duplicate record');
2103       return CRM_Import_Parser::DUPLICATE;
2104     }
2105
2106     $this->setImportStatus((int) $values[count($values) - 1], 'Imported', '');
2107     //return warning if street address is not parsed, CRM-5886
2108     return $this->processMessage($values, CRM_Import_Parser::VALID);
2109   }
2110
2111   /**
2112    * Validate a formatted contact parameter list.
2113    *
2114    * @param array $params
2115    *   Structured parameter list (as in crm_format_params).
2116    *
2117    * @throw CRM_Core_Error
2118    */
2119   public function deprecated_validate_formatted_contact(&$params): void {
2120     // Validate custom data fields
2121     if (array_key_exists('custom', $params) && is_array($params['custom'])) {
2122       foreach ($params['custom'] as $key => $custom) {
2123         if (is_array($custom)) {
2124           foreach ($custom as $fieldId => $value) {
2125             $valid = CRM_Core_BAO_CustomValue::typecheck(CRM_Utils_Array::value('type', $value),
2126               CRM_Utils_Array::value('value', $value)
2127             );
2128             if (!$valid && $value['is_required']) {
2129               throw new CRM_Core_Exception('Invalid value for custom field \'' .
2130                 $custom['name'] . '\''
2131               );
2132             }
2133             if (CRM_Utils_Array::value('type', $custom) == 'Date') {
2134               $params['custom'][$key][$fieldId]['value'] = str_replace('-', '', $params['custom'][$key][$fieldId]['value']);
2135             }
2136           }
2137         }
2138       }
2139     }
2140   }
2141
2142   /**
2143    * @param array $params
2144    * @param bool $dupeCheck
2145    * @param null|int $dedupeRuleGroupID
2146    *
2147    * @throws \CRM_Core_Exception
2148    */
2149   public function deprecated_contact_check_params(
2150     &$params,
2151     $dupeCheck = TRUE,
2152     $dedupeRuleGroupID = NULL) {
2153
2154     if (isset($params['id']) && is_numeric($params['id'])) {
2155       // @todo - ensure this is tested & remove - expectation is api call further
2156       // down validates it.
2157       if ($csType = CRM_Utils_Array::value('contact_sub_type', $params)) {
2158         if (!(CRM_Contact_BAO_ContactType::isExtendsContactType($csType, $params['contact_type']))) {
2159           throw new CRM_Core_Exception("Invalid or Mismatched Contact Subtype: " . implode(', ', (array) $csType));
2160         }
2161       }
2162     }
2163
2164     if ($dupeCheck) {
2165       // @todo switch to using api version
2166       // $dupes = civicrm_api3('Contact', 'duplicatecheck', (array('match' => $params, 'dedupe_rule_id' => $dedupeRuleGroupID)));
2167       // $ids = $dupes['count'] ? implode(',', array_keys($dupes['values'])) : NULL;
2168       $ids = CRM_Contact_BAO_Contact::getDuplicateContacts($params, $params['contact_type'], 'Unsupervised', [], CRM_Utils_Array::value('check_permissions', $params), $dedupeRuleGroupID);
2169       if ($ids != NULL) {
2170         $error = CRM_Core_Error::createError("Found matching contacts: " . implode(',', $ids),
2171           CRM_Core_Error::DUPLICATE_CONTACT,
2172           'Fatal', $ids
2173         );
2174         return civicrm_api3_create_error($error->pop());
2175       }
2176     }
2177
2178     // check for organisations with same name
2179     if (!empty($params['current_employer'])) {
2180       $organizationParams = ['organization_name' => $params['current_employer']];
2181       $dupeIds = CRM_Contact_BAO_Contact::getDuplicateContacts($organizationParams, 'Organization', 'Supervised', [], FALSE);
2182
2183       // check for mismatch employer name and id
2184       if (!empty($params['employer_id']) && !in_array($params['employer_id'], $dupeIds)
2185       ) {
2186         throw new CRM_Core_Exception('Employer name and Employer id Mismatch');
2187       }
2188
2189       // show error if multiple organisation with same name exist
2190       if (empty($params['employer_id']) && (count($dupeIds) > 1)
2191       ) {
2192         return civicrm_api3_create_error('Found more than one Organisation with same Name.');
2193       }
2194     }
2195   }
2196
2197   /**
2198    * Run import.
2199    *
2200    * @param array $mapper
2201    * @param int $mode
2202    * @param int $statusID
2203    *
2204    * @return mixed
2205    * @throws \API_Exception|\CRM_Core_Exception
2206    */
2207   public function run(
2208     $mapper = [],
2209     $mode = self::MODE_PREVIEW,
2210     $statusID = NULL
2211   ) {
2212
2213     // TODO: Make the timeout actually work
2214     $this->_onDuplicate = $onDuplicate = $this->getSubmittedValue('onDuplicate');
2215     $this->_dedupeRuleGroupID = $this->getSubmittedValue('dedupe_rule_id');
2216     // Since $this->_contactType is still being called directly do a get call
2217     // here to make sure it is instantiated.
2218     $this->getContactType();
2219     $this->getContactSubType();
2220
2221     $this->init();
2222
2223     $this->_rowCount = 0;
2224     $this->_totalCount = 0;
2225
2226     $this->_primaryKeyName = '_id';
2227     $this->_statusFieldName = '_status';
2228
2229     if ($statusID) {
2230       $this->progressImport($statusID);
2231       $startTimestamp = $currTimestamp = $prevTimestamp = time();
2232     }
2233     $dataSource = $this->getDataSourceObject();
2234     $totalRowCount = $dataSource->getRowCount(['new']);
2235     if ($mode == self::MODE_IMPORT) {
2236       $dataSource->setStatuses(['new']);
2237     }
2238     if ($this->_maxLinesToProcess > 0) {
2239       // Note this would only be the case in MapForm mode, where it is set to 100
2240       // rows. In fact mapField really only needs 2 rows - the reason for
2241       // 100 seems to be that the other import classes are processing a
2242       // csv file, and there was a concern that some rows might have more
2243       // columns than others - hence checking 100 rows perhaps seemed like
2244       // a good precaution presumably when determining the activeFieldsCount
2245       // which is the number of columns a row might have.
2246       // However, the mapField class may no longer use activeFieldsCount for contact
2247       // to be continued....
2248       $dataSource->setLimit($this->_maxLinesToProcess);
2249     }
2250
2251     while ($row = $dataSource->getRow()) {
2252       $values = array_values($row);
2253       $this->_rowCount++;
2254
2255       $this->_totalCount++;
2256
2257       if ($mode == self::MODE_MAPFIELD) {
2258         $returnCode = CRM_Import_Parser::VALID;
2259       }
2260       elseif ($mode == self::MODE_PREVIEW) {
2261         $returnCode = $this->preview($values);
2262       }
2263       elseif ($mode == self::MODE_SUMMARY) {
2264         $returnCode = $this->summary($values);
2265       }
2266       elseif ($mode == self::MODE_IMPORT) {
2267         try {
2268           $returnCode = $this->import($onDuplicate, $values);
2269         }
2270         catch (CiviCRM_API3_Exception $e) {
2271           // When we catch errors here we are not adding to the errors array - mostly
2272           // because that will become obsolete once https://github.com/civicrm/civicrm-core/pull/23292
2273           // is merged and this will replace it as the main way to handle errors (ie. update the table
2274           // and move on).
2275           $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $e->getMessage());
2276         }
2277         if ($statusID && (($this->_rowCount % 50) == 0)) {
2278           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
2279         }
2280       }
2281       else {
2282         $returnCode = self::ERROR;
2283       }
2284
2285       if ($returnCode & self::NO_MATCH) {
2286         $this->setImportStatus((int) $values[count($values) - 1], 'invalid_no_match', array_shift($values));
2287       }
2288
2289       if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
2290         $this->setImportStatus((int) $values[count($values) - 1], 'warning_unparsed_address', array_shift($values));
2291       }
2292     }
2293   }
2294
2295   /**
2296    * Given a list of the importable field keys that the user has selected.
2297    * set the active fields array to this list
2298    *
2299    * @param array $fieldKeys
2300    *   Mapped array of values.
2301    */
2302   public function setActiveFields($fieldKeys) {
2303     foreach ($fieldKeys as $key) {
2304       if (empty($this->_fields[$key])) {
2305         $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
2306       }
2307       else {
2308         $this->_activeFields[] = clone($this->_fields[$key]);
2309       }
2310     }
2311   }
2312
2313   /**
2314    * Format the field values for input to the api.
2315    *
2316    * @param array $values
2317    *   The row from the datasource.
2318    *
2319    * @return array
2320    *   Parameters mapped as described in getMappedRow
2321    *
2322    * @throws \API_Exception
2323    * @todo - clean this up a bit & merge back into `getMappedRow`
2324    *
2325    */
2326   private function getParams(array $values): array {
2327     $params = [];
2328
2329     foreach ($this->getFieldMappings() as $i => $mappedField) {
2330       // The key is in the format 5_a_b where 5 is the relationship_type_id and a_b is the direction.
2331       $relatedContactKey = $mappedField['relationship_type_id'] ? ($mappedField['relationship_type_id'] . '_' . $mappedField['relationship_direction']) : NULL;
2332       $fieldName = $mappedField['name'];
2333       $importedValue = $values[$i];
2334       if ($fieldName === 'do_not_import' || $importedValue === NULL) {
2335         continue;
2336       }
2337
2338       $locationFields = ['location_type_id', 'phone_type_id', 'provider_id', 'website_type_id'];
2339       $value = array_filter(array_intersect_key($mappedField, array_fill_keys($locationFields, 1)));
2340       if (!empty($value)) {
2341         $value[$fieldName] = $importedValue;
2342       }
2343       if (!$relatedContactKey) {
2344         if (!empty($value)) {
2345           if (!isset($params[$fieldName])) {
2346             $params[$fieldName] = [];
2347           }
2348           $params[$fieldName][] = $value;
2349         }
2350
2351         if (!isset($params[$fieldName])) {
2352           $params[$fieldName] = $importedValue;
2353         }
2354
2355       }
2356       else {
2357         if (!isset($params[$relatedContactKey])) {
2358           $params[$relatedContactKey] = ['contact_type' => $this->getRelatedContactType($mappedField['relationship_type_id'], $mappedField['relationship_direction'])];
2359         }
2360
2361         if (!empty($value)) {
2362           $params[$relatedContactKey][$fieldName][] = $importedValue ? [] : $value;
2363         }
2364         else {
2365           $params[$relatedContactKey][$fieldName] = $importedValue;
2366         }
2367       }
2368     }
2369
2370     return $params;
2371   }
2372
2373   /**
2374    * @param string $name
2375    * @param $title
2376    * @param int $type
2377    * @param string $headerPattern
2378    * @param string $dataPattern
2379    * @param bool $hasLocationType
2380    */
2381   public function addField(
2382     $name, $title, $type = CRM_Utils_Type::T_INT,
2383     $headerPattern = '//', $dataPattern = '//',
2384     $hasLocationType = FALSE
2385   ) {
2386     $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
2387     if (empty($name)) {
2388       $this->_fields['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
2389     }
2390   }
2391
2392   /**
2393    * Store parser values.
2394    *
2395    * @param CRM_Core_Session $store
2396    *
2397    * @param int $mode
2398    */
2399   public function set($store, $mode = self::MODE_SUMMARY) {
2400   }
2401
2402   /**
2403    * Export data to a CSV file.
2404    *
2405    * @param string $fileName
2406    * @param array $header
2407    * @param array $data
2408    */
2409   public static function exportCSV($fileName, $header, $data) {
2410
2411     if (file_exists($fileName) && !is_writable($fileName)) {
2412       CRM_Core_Error::movedSiteError($fileName);
2413     }
2414     //hack to remove '_status', '_statusMsg' and '_id' from error file
2415     $errorValues = [];
2416     $dbRecordStatus = ['IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW'];
2417     foreach ($data as $rowCount => $rowValues) {
2418       $count = 0;
2419       foreach ($rowValues as $key => $val) {
2420         if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
2421           break;
2422         }
2423         $errorValues[$rowCount][$key] = $val;
2424         $count++;
2425       }
2426     }
2427     $data = $errorValues;
2428
2429     $output = [];
2430     $fd = fopen($fileName, 'w');
2431
2432     foreach ($header as $key => $value) {
2433       $header[$key] = "\"$value\"";
2434     }
2435     $config = CRM_Core_Config::singleton();
2436     $output[] = implode($config->fieldSeparator, $header);
2437
2438     foreach ($data as $datum) {
2439       foreach ($datum as $key => $value) {
2440         $datum[$key] = "\"$value\"";
2441       }
2442       $output[] = implode($config->fieldSeparator, $datum);
2443     }
2444     fwrite($fd, implode("\n", $output));
2445     fclose($fd);
2446   }
2447
2448   /**
2449    * Update the status of the import row to reflect the processing outcome.
2450    *
2451    * @param int $id
2452    * @param string $status
2453    * @param string $message
2454    * @param int|null $entityID
2455    *   Optional created entity ID
2456    * @param array $relatedEntityIDs
2457    *   Optional array e.g ['related_contact' => 4]
2458    *
2459    * @throws \API_Exception
2460    * @throws \CRM_Core_Exception
2461    */
2462   public function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, array $relatedEntityIDs = []): void {
2463     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $relatedEntityIDs);
2464   }
2465
2466   /**
2467    * Format contact parameters.
2468    *
2469    * @todo this function needs re-writing & re-merging into the main function.
2470    *
2471    * Here be dragons.
2472    *
2473    * @param array $values
2474    * @param array $params
2475    *
2476    * @return bool
2477    */
2478   protected function formatContactParameters(&$values, &$params) {
2479     // Crawl through the possible classes:
2480     // Contact
2481     //      Individual
2482     //      Household
2483     //      Organization
2484     //          Location
2485     //              Address
2486     //              Email
2487     //              Phone
2488     //              IM
2489     //      Note
2490     //      Custom
2491
2492     // first add core contact values since for other Civi modules they are not added
2493     $contactFields = CRM_Contact_DAO_Contact::fields();
2494     _civicrm_api3_store_values($contactFields, $values, $params);
2495
2496     if (isset($values['contact_type'])) {
2497       // we're an individual/household/org property
2498
2499       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
2500
2501       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
2502       return TRUE;
2503     }
2504
2505     // Cache the various object fields
2506     // @todo - remove this after confirming this is just a compilation of other-wise-cached fields.
2507     static $fields = [];
2508
2509     if (isset($values['individual_prefix'])) {
2510       if (!empty($params['prefix_id'])) {
2511         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
2512         $params['prefix'] = $prefixes[$params['prefix_id']];
2513       }
2514       else {
2515         $params['prefix'] = $values['individual_prefix'];
2516       }
2517       return TRUE;
2518     }
2519
2520     if (isset($values['individual_suffix'])) {
2521       if (!empty($params['suffix_id'])) {
2522         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
2523         $params['suffix'] = $suffixes[$params['suffix_id']];
2524       }
2525       else {
2526         $params['suffix'] = $values['individual_suffix'];
2527       }
2528       return TRUE;
2529     }
2530
2531     // CRM-4575
2532     if (isset($values['email_greeting'])) {
2533       if (!empty($params['email_greeting_id'])) {
2534         $emailGreetingFilter = [
2535           'contact_type' => $params['contact_type'] ?? NULL,
2536           'greeting_type' => 'email_greeting',
2537         ];
2538         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
2539         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
2540       }
2541       else {
2542         $params['email_greeting'] = $values['email_greeting'];
2543       }
2544
2545       return TRUE;
2546     }
2547
2548     if (isset($values['postal_greeting'])) {
2549       if (!empty($params['postal_greeting_id'])) {
2550         $postalGreetingFilter = [
2551           'contact_type' => $params['contact_type'] ?? NULL,
2552           'greeting_type' => 'postal_greeting',
2553         ];
2554         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
2555         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
2556       }
2557       else {
2558         $params['postal_greeting'] = $values['postal_greeting'];
2559       }
2560       return TRUE;
2561     }
2562
2563     if (isset($values['addressee'])) {
2564       $params['addressee'] = $values['addressee'];
2565       return TRUE;
2566     }
2567
2568     if (isset($values['gender'])) {
2569       if (!empty($params['gender_id'])) {
2570         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
2571         $params['gender'] = $genders[$params['gender_id']];
2572       }
2573       else {
2574         $params['gender'] = $values['gender'];
2575       }
2576       return TRUE;
2577     }
2578
2579     if (!empty($values['preferred_communication_method'])) {
2580       $comm = [];
2581       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
2582
2583       $preffComm = explode(',', $values['preferred_communication_method']);
2584       foreach ($preffComm as $v) {
2585         $v = strtolower(trim($v));
2586         if (array_key_exists($v, $pcm)) {
2587           $comm[$pcm[$v]] = 1;
2588         }
2589       }
2590
2591       $params['preferred_communication_method'] = $comm;
2592       return TRUE;
2593     }
2594
2595     // format the website params.
2596     if (!empty($values['url'])) {
2597       static $websiteFields;
2598       if (!is_array($websiteFields)) {
2599         $websiteFields = CRM_Core_DAO_Website::fields();
2600       }
2601       if (!array_key_exists('website', $params) ||
2602         !is_array($params['website'])
2603       ) {
2604         $params['website'] = [];
2605       }
2606
2607       $websiteCount = count($params['website']);
2608       _civicrm_api3_store_values($websiteFields, $values,
2609         $params['website'][++$websiteCount]
2610       );
2611
2612       return TRUE;
2613     }
2614
2615     if (isset($values['note'])) {
2616       // add a note field
2617       if (!isset($params['note'])) {
2618         $params['note'] = [];
2619       }
2620       $noteBlock = count($params['note']) + 1;
2621
2622       $params['note'][$noteBlock] = [];
2623       if (!isset($fields['Note'])) {
2624         $fields['Note'] = CRM_Core_DAO_Note::fields();
2625       }
2626
2627       // get the current logged in civicrm user
2628       $session = CRM_Core_Session::singleton();
2629       $userID = $session->get('userID');
2630
2631       if ($userID) {
2632         $values['contact_id'] = $userID;
2633       }
2634
2635       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
2636
2637       return TRUE;
2638     }
2639
2640     // Check for custom field values
2641     $customFields = CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
2642       FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
2643     );
2644
2645     foreach ($values as $key => $value) {
2646       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
2647         // check if it's a valid custom field id
2648
2649         if (!array_key_exists($customFieldID, $customFields)) {
2650           return civicrm_api3_create_error('Invalid custom field ID');
2651         }
2652         else {
2653           $params[$key] = $value;
2654         }
2655       }
2656     }
2657     return TRUE;
2658   }
2659
2660   /**
2661    * Format location block ready for importing.
2662    *
2663    * There is some test coverage for this in CRM_Contact_Import_Parser_ContactTest
2664    * e.g. testImportPrimaryAddress.
2665    *
2666    * @param array $values
2667    * @param array $params
2668    *
2669    * @return bool
2670    */
2671   protected function formatLocationBlock(&$values, &$params) {
2672     $blockTypes = [
2673       'phone' => 'Phone',
2674       'email' => 'Email',
2675       'im' => 'IM',
2676       'openid' => 'OpenID',
2677       'phone_ext' => 'Phone',
2678     ];
2679     foreach ($blockTypes as $blockFieldName => $block) {
2680       if (!array_key_exists($blockFieldName, $values)) {
2681         continue;
2682       }
2683       $blockIndex = $values['location_type_id'] . (!empty($values['phone_type_id']) ? '_' . $values['phone_type_id'] : '');
2684
2685       // block present in value array.
2686       if (!array_key_exists($blockFieldName, $params) || !is_array($params[$blockFieldName])) {
2687         $params[$blockFieldName] = [];
2688       }
2689
2690       $fields[$block] = $this->getMetadataForEntity($block);
2691
2692       // copy value to dao field name.
2693       if ($blockFieldName == 'im') {
2694         $values['name'] = $values[$blockFieldName];
2695       }
2696
2697       _civicrm_api3_store_values($fields[$block], $values,
2698         $params[$blockFieldName][$blockIndex]
2699       );
2700
2701       $this->fillPrimary($params[$blockFieldName][$blockIndex], $values, $block, CRM_Utils_Array::value('id', $params));
2702
2703       if (empty($params['id']) && (count($params[$blockFieldName]) == 1)) {
2704         $params[$blockFieldName][$blockIndex]['is_primary'] = TRUE;
2705       }
2706
2707       // we only process single block at a time.
2708       return TRUE;
2709     }
2710
2711     // handle address fields.
2712     if (!array_key_exists('address', $params) || !is_array($params['address'])) {
2713       $params['address'] = [];
2714     }
2715
2716     // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
2717     // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
2718     // the address in CRM_Core_BAO_Address::create method
2719     if (!empty($values['location_type_id'])) {
2720       static $customFields = [];
2721       if (empty($customFields)) {
2722         $customFields = CRM_Core_BAO_CustomField::getFields('Address');
2723       }
2724       // make a copy of values, as we going to make changes
2725       $newValues = $values;
2726       foreach ($values as $key => $val) {
2727         $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
2728         if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
2729
2730           $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
2731           if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
2732             $mulValues = explode(',', $val);
2733             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
2734             $newValues[$key] = [];
2735             foreach ($mulValues as $v1) {
2736               foreach ($customOption as $v2) {
2737                 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
2738                   (strtolower($v2['value']) == strtolower(trim($v1)))
2739                 ) {
2740                   if ($htmlType == 'CheckBox') {
2741                     $newValues[$key][$v2['value']] = 1;
2742                   }
2743                   else {
2744                     $newValues[$key][] = $v2['value'];
2745                   }
2746                 }
2747               }
2748             }
2749           }
2750         }
2751       }
2752       // consider new values
2753       $values = $newValues;
2754     }
2755
2756     $fields['Address'] = $this->getMetadataForEntity('Address');
2757     // @todo this is kinda replicated below....
2758     _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$values['location_type_id']]);
2759
2760     $addressFields = [
2761       'county',
2762       'country',
2763       'state_province',
2764       'supplemental_address_1',
2765       'supplemental_address_2',
2766       'supplemental_address_3',
2767       'StateProvince.name',
2768     ];
2769     foreach (array_keys($customFields) as $customFieldID) {
2770       $addressFields[] = 'custom_' . $customFieldID;
2771     }
2772
2773     foreach ($addressFields as $field) {
2774       if (array_key_exists($field, $values)) {
2775         if (!array_key_exists('address', $params)) {
2776           $params['address'] = [];
2777         }
2778         $params['address'][$values['location_type_id']][$field] = $values[$field];
2779       }
2780     }
2781
2782     $this->fillPrimary($params['address'][$values['location_type_id']], $values, 'address', CRM_Utils_Array::value('id', $params));
2783     return TRUE;
2784   }
2785
2786   /**
2787    * Get the field metadata for the relevant entity.
2788    *
2789    * @param string $entity
2790    *
2791    * @return array
2792    */
2793   protected function getMetadataForEntity($entity) {
2794     if (!isset($this->fieldMetadata[$entity])) {
2795       $className = "CRM_Core_DAO_$entity";
2796       $this->fieldMetadata[$entity] = $className::fields();
2797     }
2798     return $this->fieldMetadata[$entity];
2799   }
2800
2801   /**
2802    * Fill in the primary location.
2803    *
2804    * If the contact has a primary address we update it. Otherwise
2805    * we add an address of the default location type.
2806    *
2807    * @param array $params
2808    *   Address block parameters
2809    * @param array $values
2810    *   Input values
2811    * @param string $entity
2812    *  - address, email, phone
2813    * @param int|null $contactID
2814    *
2815    * @throws \CiviCRM_API3_Exception
2816    */
2817   protected function fillPrimary(&$params, $values, $entity, $contactID) {
2818     if ($values['location_type_id'] === 'Primary') {
2819       if ($contactID) {
2820         $primary = civicrm_api3($entity, 'get', [
2821           'return' => 'location_type_id',
2822           'contact_id' => $contactID,
2823           'is_primary' => 1,
2824           'sequential' => 1,
2825         ]);
2826       }
2827       $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
2828       $params['location_type_id'] = (int) (isset($primary) && $primary['count']) ? $primary['values'][0]['location_type_id'] : $defaultLocationType->id;
2829       $params['is_primary'] = 1;
2830     }
2831   }
2832
2833   /**
2834    * Get the civicrm_mapping_field appropriate layout for the mapper input.
2835    *
2836    * The input looks something like ['street_address', 1]
2837    * and would be mapped to ['name' => 'street_address', 'location_type_id' =>
2838    * 1]
2839    *
2840    * @param array $fieldMapping
2841    * @param int $mappingID
2842    * @param int $columnNumber
2843    *
2844    * @return array
2845    * @throws \API_Exception
2846    */
2847   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
2848     $isRelationshipField = preg_match('/\d*_a_b|b_a$/', $fieldMapping[0]);
2849     $fieldName = $isRelationshipField ? $fieldMapping[1] : $fieldMapping[0];
2850     $locationTypeID = NULL;
2851     $possibleLocationField = $isRelationshipField ? 2 : 1;
2852     if ($fieldName !== 'url' && is_numeric($fieldMapping[$possibleLocationField] ?? NULL)) {
2853       $locationTypeID = $fieldMapping[$possibleLocationField];
2854     }
2855     return [
2856       'name' => $fieldName,
2857       'mapping_id' => $mappingID,
2858       'relationship_type_id' => $isRelationshipField ? substr($fieldMapping[0], 0, -4) : NULL,
2859       'relationship_direction' => $isRelationshipField ? substr($fieldMapping[0], -3) : NULL,
2860       'column_number' => $columnNumber,
2861       'contact_type' => $this->getContactType(),
2862       'website_type_id' => $fieldName !== 'url' ? NULL : ($isRelationshipField ? $fieldMapping[2] : $fieldMapping[1]),
2863       'phone_type_id' => $fieldName !== 'phone' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
2864       'im_provider_id' => $fieldName !== 'im' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
2865       'location_type_id' => $locationTypeID,
2866     ];
2867   }
2868
2869   /**
2870    * @param array $mappedField
2871    *   Field detail as would be saved in field_mapping table
2872    *   or as returned from getMappingFieldFromMapperInput
2873    *
2874    * @return string
2875    * @throws \API_Exception
2876    */
2877   public function getMappedFieldLabel(array $mappedField): string {
2878     $this->setFieldMetadata();
2879     $title = [];
2880     if ($mappedField['relationship_type_id']) {
2881       $title[] = $this->getRelationshipLabel($mappedField['relationship_type_id'], $mappedField['relationship_direction']);
2882     }
2883     $title[] = $this->getImportableFieldsMetadata()[$mappedField['name']]['title'];
2884     if ($mappedField['location_type_id']) {
2885       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Address', 'location_type_id', $mappedField['location_type_id']);
2886     }
2887     if ($mappedField['website_type_id']) {
2888       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Website', 'website_type_id', $mappedField['website_type_id']);
2889     }
2890     if ($mappedField['phone_type_id']) {
2891       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $mappedField['phone_type_id']);
2892     }
2893     if ($mappedField['im_provider_id']) {
2894       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $mappedField['provider_id']);
2895     }
2896     return implode(' - ', $title);
2897   }
2898
2899   /**
2900    * Get the relevant label for the relationship.
2901    *
2902    * @param int $id
2903    * @param string $direction
2904    *
2905    * @return string
2906    * @throws \API_Exception
2907    */
2908   protected function getRelationshipLabel(int $id, string $direction): string {
2909     if (empty($this->relationshipLabels[$id . $direction])) {
2910       $this->relationshipLabels[$id . $direction] =
2911       $fieldName = 'label_' . $direction;
2912       $this->relationshipLabels[$id . $direction] = (string) RelationshipType::get(FALSE)
2913         ->addWhere('id', '=', $id)
2914         ->addSelect($fieldName)->execute()->first()[$fieldName];
2915     }
2916     return $this->relationshipLabels[$id . $direction];
2917   }
2918
2919   /**
2920    * Transform the input parameters into the form handled by the input routine.
2921    *
2922    * @param array $values
2923    *   Input parameters as they come in from the datasource
2924    *   eg. ['Bob', 'Smith', 'bob@example.org', '123-456']
2925    *
2926    * @return array
2927    *   Parameters mapped to CiviCRM fields based on the mapping
2928    *   and specified contact type. eg.
2929    *   [
2930    *     'contact_type' => 'Individual',
2931    *     'first_name' => 'Bob',
2932    *     'last_name' => 'Smith',
2933    *     'phone' => ['phone' => '123', 'location_type_id' => 1, 'phone_type_id' => 1],
2934    *     '5_a_b' => ['contact_type' => 'Organization', 'url' => ['url' => 'https://example.org', 'website_type_id' => 1]]
2935    *     'im' => ['im' => 'my-handle', 'location_type_id' => 1, 'provider_id' => 1],
2936    *
2937    * @throws \API_Exception
2938    */
2939   public function getMappedRow(array $values): array {
2940     $params = $this->getParams($values);
2941     $params['contact_type'] = $this->getContactType();
2942     return $params;
2943   }
2944
2945   /**
2946    * Validate the import values.
2947    *
2948    * The values array represents a row in the datasource.
2949    *
2950    * @param array $values
2951    *
2952    * @throws \API_Exception
2953    * @throws \CRM_Core_Exception
2954    */
2955   public function validateValues(array $values): void {
2956     $params = $this->getMappedRow($values);
2957     $this->validateRequiredContactFields($params['contact_type'], $params, $this->isUpdateExistingContacts());
2958     foreach ($params as $key => $value) {
2959       // If the key is a relationship key - eg. 5_a_b or 10_b_a
2960       // then the value is an array that describes an existing contact.
2961       // We need to check the fields are present to identify or create this
2962       // contact.
2963       if (preg_match('/^\d+_[a|b]_[a|b]$/', $key)) {
2964         $this->validateRequiredContactFields($value['contact_type'], $value, TRUE, '(' . $this->getRelatedContactLabel(substr($key, 0, -4), substr($key, -3)) . ')');
2965       }
2966     }
2967
2968     //check for duplicate external Identifier
2969     $externalID = $params['external_identifier'] ?? NULL;
2970     if ($externalID) {
2971       /* If it's a dupe,external Identifier  */
2972
2973       if ($externalDupe = CRM_Utils_Array::value($externalID, $this->_allExternalIdentifiers)) {
2974         $errorMessage = ts('External ID conflicts with record %1', [1 => $externalDupe]);
2975         throw new CRM_Core_Exception($errorMessage);
2976       }
2977       //otherwise, count it and move on
2978       $this->_allExternalIdentifiers[$externalID] = $this->_lineCount;
2979     }
2980
2981     //date-format part ends
2982
2983     $errorMessage = NULL;
2984
2985     //CRM-5125
2986     //add custom fields for contact sub type
2987     $csType = NULL;
2988     if (!empty($this->_contactSubType)) {
2989       $csType = $this->_contactSubType;
2990     }
2991
2992     //checking error in custom data
2993     $this->isErrorInCustomData($params, $errorMessage, $csType, $this->_relationships);
2994
2995     //checking error in core data
2996     $this->isErrorInCoreData($params, $errorMessage);
2997     if ($errorMessage) {
2998       $tempMsg = "Invalid value for field(s) : $errorMessage";
2999       throw new CRM_Core_Exception($tempMsg);
3000     }
3001   }
3002
3003   /**
3004    * Get the field mappings for the import.
3005    *
3006    * This is the same format as saved in civicrm_mapping_field except
3007    * that location_type_id = 'Primary' rather than empty where relevant.
3008    * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
3009    *
3010    * @return array
3011    * @throws \API_Exception
3012    */
3013   protected function getFieldMappings(): array {
3014     $mappedFields = [];
3015     foreach ($this->getSubmittedValue('mapper') as $i => $mapperRow) {
3016       $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
3017       if (!$mappedField['location_type_id'] && !empty($this->importableFieldsMetadata[$mappedField['name']]['hasLocationType'])) {
3018         $mappedField['location_type_id'] = 'Primary';
3019       }
3020       // Just for clarity since 0 is a pseudo-value
3021       unset($mappedField['mapping_id']);
3022       // Annoyingly the civicrm_mapping_field name for this differs from civicrm_im.
3023       // Test cover in `CRM_Contact_Import_Parser_ContactTest::testMapFields`
3024       $mappedField['provider_id'] = $mappedField['im_provider_id'];
3025       unset($mappedField['im_provider_id']);
3026       $mappedFields[] = $mappedField;
3027     }
3028     return $mappedFields;
3029   }
3030
3031   /**
3032    * Get the related contact type.
3033    *
3034    * @param int|null $relationshipTypeID
3035    * @param int|string $relationshipDirection
3036    *
3037    * @return null|string
3038    *
3039    * @throws \API_Exception
3040    */
3041   protected function getRelatedContactType($relationshipTypeID, $relationshipDirection): ?string {
3042     if (!$relationshipTypeID) {
3043       return NULL;
3044     }
3045     $relationshipField = 'contact_type_' . substr($relationshipDirection, -1);
3046     return $this->getRelationshipType($relationshipTypeID, $relationshipDirection)[$relationshipField];
3047   }
3048
3049   /**
3050    * Get the related contact type.
3051    *
3052    * @param int|null $relationshipTypeID
3053    * @param int|string $relationshipDirection
3054    *
3055    * @return null|string
3056    *
3057    * @throws \API_Exception
3058    */
3059   protected function getRelatedContactLabel($relationshipTypeID, $relationshipDirection): ?string {
3060     $relationshipField = 'label_' . $relationshipDirection;
3061     return $this->getRelationshipType($relationshipTypeID, $relationshipDirection)[$relationshipField];
3062   }
3063
3064   /**
3065    * Get the relationship type.
3066    *
3067    * @param int $relationshipTypeID
3068    *
3069    * @return string[]
3070    * @throws \API_Exception
3071    */
3072   protected function getRelationshipType(int $relationshipTypeID): array {
3073     $cacheKey = 'relationship_type' . $relationshipTypeID;
3074     if (!isset(Civi::$statics[__CLASS__][$cacheKey])) {
3075       Civi::$statics[__CLASS__][$cacheKey] = RelationshipType::get(FALSE)
3076         ->addWhere('id', '=', $relationshipTypeID)
3077         ->addSelect('*')->execute()->first();
3078     }
3079     return Civi::$statics[__CLASS__][$cacheKey];
3080   }
3081
3082 }