CRM/Contact/Import/Parser/Contact.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Contact;
  13 use Civi\Api4\RelationshipType;
  14
  15 require_once 'CRM/Utils/DeprecatedUtils.php';
  16 require_once 'api/v3/utils.php';
  17
  18 /**
  19  *
  20  * @package CRM
  21  * @copyright CiviCRM LLC https://civicrm.org/licensing
  22  */
  23
  24 /**
  25  * class to parse contact csv files
  26  */
  27 class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser {
  28
  29   use CRM_Contact_Import_MetadataTrait;
  30
  31   protected $_mapperKeys = [];
  32   protected $_mapperLocType = [];
  33   protected $_mapperPhoneType;
  34   protected $_mapperImProvider;
  35   protected $_mapperWebsiteType;
  36   protected $_mapperRelated;
  37   protected $_mapperRelatedContactType;
  38   protected $_mapperRelatedContactDetails;
  39   protected $_mapperRelatedContactEmailType;
  40   protected $_mapperRelatedContactImProvider;
  41   protected $_mapperRelatedContactWebsiteType;
  42   protected $_relationships;
  43
  44   protected $_emailIndex;
  45   protected $_firstNameIndex;
  46   protected $_lastNameIndex;
  47
  48   protected $_householdNameIndex;
  49   protected $_organizationNameIndex;
  50
  51   protected $_phoneIndex;
  52
  53   /**
  54    * Is update only permitted on an id match.
  55    *
  56    * Note this historically was true for when id or external identifier was
  57    * present. However, CRM-17275 determined that a dedupe-match could over-ride
  58    * external identifier.
  59    *
  60    * @var bool
  61    */
  62   protected $_updateWithId;
  63   protected $_retCode;
  64
  65   protected $_externalIdentifierIndex;
  66   protected $_allExternalIdentifiers;
  67   protected $_parseStreetAddress;
  68
  69   /**
  70    * Array of successfully imported contact id's
  71    *
  72    * @var array
  73    */
  74   protected $_newContacts;
  75
  76   /**
  77    * Line count id.
  78    *
  79    * @var int
  80    */
  81   protected $_lineCount;
  82
  83   /**
  84    * Array of successfully imported related contact id's
  85    *
  86    * @var array
  87    */
  88   protected $_newRelatedContacts;
  89
  90   /**
  91    * Array of all the contacts whose street addresses are not parsed.
  92    * of this import process
  93    * @var array
  94    */
  95   protected $_unparsedStreetAddressContacts;
  96
  97   protected $_tableName;
  98
  99   /**
 100    * Total number of lines in file
 101    *
 102    * @var int
 103    */
 104   protected $_rowCount;
 105
 106   /**
 107    * Running total number of un-matched Contacts.
 108    *
 109    * @var int
 110    */
 111   protected $_unMatchCount;
 112
 113   /**
 114    * Array of unmatched lines.
 115    *
 116    * @var array
 117    */
 118   protected $_unMatch;
 119
 120   /**
 121    * Total number of contacts with unparsed addresses
 122    * @var int
 123    */
 124   protected $_unparsedAddressCount;
 125
 126   /**
 127    * Filename of mismatch data
 128    *
 129    * @var string
 130    */
 131   protected $_misMatchFilemName;
 132
 133   protected $_primaryKeyName;
 134   protected $_statusFieldName;
 135
 136   protected $fieldMetadata = [];
 137
 138   /**
 139    * Relationship labels.
 140    *
 141    * Temporary cache of labels to reduce queries in getRelationshipLabels.
 142    *
 143    * @var array
 144    *   e.g ['5a_b' => 'Employer', '5b_a' => 'Employee']
 145    */
 146   protected $relationshipLabels = [];
 147
 148   /**
 149    * On duplicate
 150    *
 151    * @var int
 152    */
 153   public $_onDuplicate;
 154
 155   /**
 156    * Dedupe rule group id to use if set
 157    *
 158    * @var int
 159    */
 160   public $_dedupeRuleGroupID = NULL;
 161
 162   /**
 163    * Class constructor.
 164    *
 165    * @param array $mapperKeys
 166    * @param array $mapperLocType
 167    * @param array $mapperPhoneType
 168    * @param array $mapperImProvider
 169    * @param array $mapperRelated
 170    * @param array $mapperRelatedContactType
 171    * @param array $mapperRelatedContactDetails
 172    * @param array $mapperRelatedContactLocType
 173    * @param array $mapperRelatedContactPhoneType
 174    * @param array $mapperRelatedContactImProvider
 175    * @param array $mapperWebsiteType
 176    * @param array $mapperRelatedContactWebsiteType
 177    */
 178   public function __construct(
 179     $mapperKeys = [], $mapperLocType = [], $mapperPhoneType = [], $mapperImProvider = [], $mapperRelated = [], $mapperRelatedContactType = [], $mapperRelatedContactDetails = [], $mapperRelatedContactLocType = [], $mapperRelatedContactPhoneType = [], $mapperRelatedContactImProvider = [],
 180     $mapperWebsiteType = [], $mapperRelatedContactWebsiteType = []
 181   ) {
 182     parent::__construct();
 183     $this->_mapperKeys = $mapperKeys;
 184     $this->_mapperLocType = &$mapperLocType;
 185     $this->_mapperPhoneType = &$mapperPhoneType;
 186     $this->_mapperWebsiteType = $mapperWebsiteType;
 187     // get IM service provider type id for contact
 188     $this->_mapperImProvider = &$mapperImProvider;
 189     $this->_mapperRelated = &$mapperRelated;
 190     $this->_mapperRelatedContactType = &$mapperRelatedContactType;
 191     $this->_mapperRelatedContactDetails = &$mapperRelatedContactDetails;
 192     $this->_mapperRelatedContactLocType = &$mapperRelatedContactLocType;
 193     $this->_mapperRelatedContactPhoneType = &$mapperRelatedContactPhoneType;
 194     $this->_mapperRelatedContactWebsiteType = $mapperRelatedContactWebsiteType;
 195     // get IM service provider type id for related contact
 196     $this->_mapperRelatedContactImProvider = &$mapperRelatedContactImProvider;
 197   }
 198
 199   /**
 200    * The initializer code, called before processing.
 201    */
 202   public function init() {
 203     $this->setFieldMetadata();
 204     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 205       $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field));
 206     }
 207     $this->_newContacts = [];
 208
 209     $this->setActiveFields($this->_mapperKeys);
 210     $this->setActiveFieldLocationTypes($this->_mapperLocType);
 211     $this->setActiveFieldPhoneTypes($this->_mapperPhoneType);
 212     $this->setActiveFieldWebsiteTypes($this->_mapperWebsiteType);
 213     //set active fields of IM provider of contact
 214     $this->setActiveFieldImProviders($this->_mapperImProvider);
 215
 216     //related info
 217     $this->setActiveFieldRelated($this->_mapperRelated);
 218     $this->setActiveFieldRelatedContactType($this->_mapperRelatedContactType);
 219     $this->setActiveFieldRelatedContactDetails($this->_mapperRelatedContactDetails);
 220     $this->setActiveFieldRelatedContactLocType($this->_mapperRelatedContactLocType);
 221     $this->setActiveFieldRelatedContactPhoneType($this->_mapperRelatedContactPhoneType);
 222     $this->setActiveFieldRelatedContactWebsiteType($this->_mapperRelatedContactWebsiteType);
 223     //set active fields of IM provider of related contact
 224     $this->setActiveFieldRelatedContactImProvider($this->_mapperRelatedContactImProvider);
 225
 226     $this->_phoneIndex = -1;
 227     $this->_emailIndex = -1;
 228     $this->_firstNameIndex = -1;
 229     $this->_lastNameIndex = -1;
 230     $this->_householdNameIndex = -1;
 231     $this->_organizationNameIndex = -1;
 232     $this->_externalIdentifierIndex = -1;
 233
 234     $index = 0;
 235     foreach ($this->_mapperKeys as $key) {
 236       if (substr($key, 0, 5) == 'email' && substr($key, 0, 14) != 'email_greeting') {
 237         $this->_emailIndex = $index;
 238       }
 239       if (substr($key, 0, 5) == 'phone') {
 240         $this->_phoneIndex = $index;
 241       }
 242       if ($key == 'first_name') {
 243         $this->_firstNameIndex = $index;
 244       }
 245       if ($key == 'last_name') {
 246         $this->_lastNameIndex = $index;
 247       }
 248       if ($key == 'household_name') {
 249         $this->_householdNameIndex = $index;
 250       }
 251       if ($key == 'organization_name') {
 252         $this->_organizationNameIndex = $index;
 253       }
 254
 255       if ($key == 'external_identifier') {
 256         $this->_externalIdentifierIndex = $index;
 257         $this->_allExternalIdentifiers = [];
 258       }
 259       $index++;
 260     }
 261
 262     $this->_updateWithId = FALSE;
 263     if (in_array('id', $this->_mapperKeys) || ($this->_externalIdentifierIndex >= 0 && in_array($this->_onDuplicate, [
 264       CRM_Import_Parser::DUPLICATE_UPDATE,
 265       CRM_Import_Parser::DUPLICATE_FILL,
 266     ]))) {
 267       $this->_updateWithId = TRUE;
 268     }
 269
 270     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
 271   }
 272
 273   /**
 274    * Gets the fields available for importing in a key-name, title format.
 275    *
 276    * @return array
 277    *   eg. ['first_name' => 'First Name'.....]
 278    *
 279    * @throws \API_Exception
 280    *
 281    * @todo - we are constructing the metadata before we
 282    * have set the contact type so we re-do it here.
 283    *
 284    * Once we have cleaned up the way the mapper is handled
 285    * we can ditch all the existing _construct parameters in favour
 286    * of just the userJobID - there are current open PRs towards this end.
 287    */
 288   public function getAvailableFields(): array {
 289     $this->setFieldMetadata();
 290     $return = [];
 291     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 292       if ($name === 'id' && $this->isSkipDuplicates()) {
 293         // Duplicates are being skipped so id matching is not availble.
 294         continue;
 295       }
 296       $return[$name] = $field['title'];
 297     }
 298     return $return;
 299   }
 300
 301   /**
 302    * Did the user specify duplicates should be skipped and not imported.
 303    *
 304    * @return bool
 305    *
 306    * @throws \API_Exception
 307    */
 308   private function isSkipDuplicates(): bool {
 309     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 310   }
 311
 312   /**
 313    * Handle the values in preview mode.
 314    *
 315    * @param array $values
 316    *   The array of values belonging to this line.
 317    *
 318    * @return bool
 319    *   the result of this processing
 320    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 321    */
 322   public function preview(&$values) {
 323     return $this->summary($values);
 324   }
 325
 326   /**
 327    * Handle the values in summary mode.
 328    *
 329    * @param array $values
 330    *   The array of values belonging to this line.
 331    *
 332    * @return int
 333    *   the result of this processing
 334    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 335    */
 336   public function summary(&$values): int {
 337     $params = $this->getMappedRow($values);
 338     $rowNumber = (int) ($values[count($values) - 1]);
 339     try {
 340       $errorMessage = NULL;
 341       $errorRequired = FALSE;
 342       switch ($this->_contactType) {
 343         case 'Individual':
 344           $missingNames = [];
 345           if ($this->_firstNameIndex < 0 || empty($values[$this->_firstNameIndex])) {
 346             $errorRequired = TRUE;
 347             $missingNames[] = ts('First Name');
 348           }
 349           if ($this->_lastNameIndex < 0 || empty($values[$this->_lastNameIndex])) {
 350             $errorRequired = TRUE;
 351             $missingNames[] = ts('Last Name');
 352           }
 353           if ($errorRequired) {
 354             $and = ' ' . ts('and') . ' ';
 355             $errorMessage = ts('Missing required fields:') . ' ' . implode($and, $missingNames);
 356           }
 357           break;
 358
 359         case 'Household':
 360           if ($this->_householdNameIndex < 0 || empty($values[$this->_householdNameIndex])) {
 361             $errorRequired = TRUE;
 362             $errorMessage = ts('Missing required fields:') . ' ' . ts('Household Name');
 363           }
 364           break;
 365
 366         case 'Organization':
 367           if ($this->_organizationNameIndex < 0 || empty($values[$this->_organizationNameIndex])) {
 368             $errorRequired = TRUE;
 369             $errorMessage = ts('Missing required fields:') . ' ' . ts('Organization Name');
 370           }
 371           break;
 372       }
 373
 374       if ($this->_emailIndex >= 0) {
 375         /* If we don't have the required fields, bail */
 376
 377         if ($this->_contactType === 'Individual' && !$this->_updateWithId) {
 378           if ($errorRequired && empty($values[$this->_emailIndex])) {
 379             if ($errorMessage) {
 380               $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
 381             }
 382             else {
 383               $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
 384             }
 385             throw new CRM_Core_Exception($errorMessage);
 386           }
 387         }
 388
 389         $email = $values[$this->_emailIndex] ?? NULL;
 390         if ($email) {
 391           if (!CRM_Utils_Rule::email($email)) {
 392             throw new CRM_Core_Exception($errorMessage);
 393           }
 394         }
 395       }
 396       elseif ($errorRequired && !$this->_updateWithId) {
 397         if ($errorMessage) {
 398           $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
 399         }
 400         else {
 401           $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
 402         }
 403         throw new CRM_Core_Exception($errorMessage);
 404       }
 405
 406       //check for duplicate external Identifier
 407       $externalID = $values[$this->_externalIdentifierIndex] ?? NULL;
 408       if ($externalID) {
 409         /* If it's a dupe,external Identifier  */
 410
 411         if ($externalDupe = CRM_Utils_Array::value($externalID, $this->_allExternalIdentifiers)) {
 412           $errorMessage = ts('External ID conflicts with record %1', [1 => $externalDupe]);
 413           throw new CRM_Core_Exception($errorMessage);
 414         }
 415         //otherwise, count it and move on
 416         $this->_allExternalIdentifiers[$externalID] = $this->_lineCount;
 417       }
 418
 419       //date-format part ends
 420
 421       $errorMessage = NULL;
 422
 423       //CRM-5125
 424       //add custom fields for contact sub type
 425       $csType = NULL;
 426       if (!empty($this->_contactSubType)) {
 427         $csType = $this->_contactSubType;
 428       }
 429
 430       //checking error in custom data
 431       $this->isErrorInCustomData($params, $errorMessage, $csType, $this->_relationships);
 432
 433       //checking error in core data
 434       $this->isErrorInCoreData($params, $errorMessage);
 435       if ($errorMessage) {
 436         $tempMsg = "Invalid value for field(s) : $errorMessage";
 437         throw new CRM_Core_Exception($tempMsg);
 438       }
 439     }
 440     catch (CRM_Core_Exception $e) {
 441       $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
 442       array_unshift($values, $e->getMessage());
 443       return CRM_Import_Parser::ERROR;
 444     }
 445     $this->setImportStatus($rowNumber, 'NEW', '');
 446
 447     return CRM_Import_Parser::VALID;
 448   }
 449
 450   /**
 451    * Get Array of all the fields that could potentially be part
 452    * import process
 453    *
 454    * @return array
 455    */
 456   public function getAllFields() {
 457     return $this->_fields;
 458   }
 459
 460   /**
 461    * Handle the values in import mode.
 462    *
 463    * @param int $onDuplicate
 464    *   The code for what action to take on duplicates.
 465    * @param array $values
 466    *   The array of values belonging to this line.
 467    *
 468    * @param bool $doGeocodeAddress
 469    *
 470    * @return bool
 471    *   the result of this processing
 472    *
 473    * @throws \CiviCRM_API3_Exception
 474    * @throws \CRM_Core_Exception
 475    * @throws \API_Exception
 476    */
 477   public function import($onDuplicate, &$values, $doGeocodeAddress = FALSE) {
 478     $this->_unparsedStreetAddressContacts = [];
 479     if (!$doGeocodeAddress) {
 480       // CRM-5854, reset the geocode method to null to prevent geocoding
 481       CRM_Utils_GeocodeProvider::disableForSession();
 482     }
 483
 484     // first make sure this is a valid line
 485     //$this->_updateWithId = false;
 486     $response = $this->summary($values);
 487     $statusFieldName = $this->_statusFieldName;
 488
 489     if ($response != CRM_Import_Parser::VALID) {
 490       $this->setImportStatus((int) $values[count($values) - 1], 'Invalid', "Invalid (Error Code: $response)");
 491       return $response;
 492     }
 493
 494     $params = &$this->getActiveFieldParams();
 495     $formatted = [
 496       'contact_type' => $this->_contactType,
 497     ];
 498
 499     $contactFields = CRM_Contact_DAO_Contact::import();
 500
 501     //check if external identifier exists in database
 502     if (!empty($params['external_identifier']) && (!empty($params['id']) || in_array($onDuplicate, [
 503       CRM_Import_Parser::DUPLICATE_SKIP,
 504       CRM_Import_Parser::DUPLICATE_NOCHECK,
 505     ]))) {
 506
 507       $extIDResult = civicrm_api3('Contact', 'get', [
 508         'external_identifier' => $params['external_identifier'],
 509         'showAll' => 'all',
 510         'return' => ['id', 'contact_is_deleted'],
 511       ]);
 512       if (isset($extIDResult['id'])) {
 513         // record with matching external identifier does exist.
 514         $internalCid = $extIDResult['id'];
 515         if ($internalCid != CRM_Utils_Array::value('id', $params)) {
 516           if ($extIDResult['values'][$internalCid]['contact_is_deleted'] == 1) {
 517             // And it is deleted. What to do? If we skip it, they user
 518             // will be under the impression that the record exists in
 519             // the database, yet they won't be able to find it. If we
 520             // don't skip it, the database will try to insert a new record
 521             // with an external_identifier that is non-unique. So...
 522             // we will update this contact to remove the external_identifier
 523             // and let a new record be created.
 524             $update_params = ['id' => $internalCid, 'external_identifier' => ''];
 525             civicrm_api3('Contact', 'create', $update_params);
 526           }
 527           else {
 528             $errorMessage = ts('External ID already exists in Database.');
 529             array_unshift($values, $errorMessage);
 530             $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 531             return CRM_Import_Parser::DUPLICATE;
 532           }
 533         }
 534       }
 535     }
 536
 537     if (!empty($this->_contactSubType)) {
 538       $params['contact_sub_type'] = $this->_contactSubType;
 539     }
 540
 541     if ($subType = CRM_Utils_Array::value('contact_sub_type', $params)) {
 542       if (CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType, FALSE, 'label')) {
 543         $subTypes = CRM_Contact_BAO_ContactType::subTypePairs($this->_contactType, FALSE, NULL);
 544         $params['contact_sub_type'] = array_search($subType, $subTypes);
 545       }
 546       elseif (!CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType)) {
 547         $message = "Mismatched or Invalid Contact Subtype.";
 548         array_unshift($values, $message);
 549         return CRM_Import_Parser::NO_MATCH;
 550       }
 551     }
 552
 553     // Get contact id to format common data in update/fill mode,
 554     // prioritising a dedupe rule check over an external_identifier check, but falling back on ext id.
 555     if ($this->_updateWithId && empty($params['id'])) {
 556       try {
 557         $possibleMatches = $this->getPossibleContactMatches($params);
 558       }
 559       catch (CRM_Core_Exception $e) {
 560         $errorMessage = $e->getMessage();
 561         array_unshift($values, $errorMessage);
 562
 563         $importRecordParams = [
 564           $statusFieldName => 'ERROR',
 565           "${statusFieldName}Msg" => $errorMessage,
 566         ];
 567         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 568         return CRM_Import_Parser::ERROR;
 569       }
 570       foreach ($possibleMatches as $possibleID) {
 571         $params['id'] = $formatted['id'] = $possibleID;
 572       }
 573     }
 574     //format common data, CRM-4062
 575     $this->formatCommonData($params, $formatted, $contactFields);
 576
 577     $relationship = FALSE;
 578     $createNewContact = TRUE;
 579     // Support Match and Update Via Contact ID
 580     if ($this->_updateWithId && isset($params['id'])) {
 581       $createNewContact = FALSE;
 582       // @todo - it feels like all the rows from here to the end of the IF
 583       // could be removed in favour of a simple check for whether the contact_type & id match
 584       $matchedIDs = $this->getIdsOfMatchingContacts($formatted);
 585       if (!empty($matchedIDs)) {
 586         if (count($matchedIDs) >= 1) {
 587           $updateflag = TRUE;
 588           foreach ($matchedIDs as $contactId) {
 589             if ($params['id'] == $contactId) {
 590               $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 591               if ($formatted['contact_type'] == $contactType) {
 592                 //validation of subtype for update mode
 593                 //CRM-5125
 594                 $contactSubType = NULL;
 595                 if (!empty($params['contact_sub_type'])) {
 596                   $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 597                 }
 598
 599                 if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 600
 601                   $message = "Mismatched contact SubTypes :";
 602                   array_unshift($values, $message);
 603                   $updateflag = FALSE;
 604                   $this->_retCode = CRM_Import_Parser::NO_MATCH;
 605                 }
 606                 else {
 607                   $updateflag = FALSE;
 608                   $this->_retCode = CRM_Import_Parser::VALID;
 609                 }
 610               }
 611               else {
 612                 $message = "Mismatched contact Types :";
 613                 array_unshift($values, $message);
 614                 $updateflag = FALSE;
 615                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 616               }
 617             }
 618           }
 619           if ($updateflag) {
 620             $message = "Mismatched contact IDs OR Mismatched contact Types :";
 621             array_unshift($values, $message);
 622             $this->_retCode = CRM_Import_Parser::NO_MATCH;
 623           }
 624         }
 625       }
 626       else {
 627         $contactType = NULL;
 628         if (!empty($params['id'])) {
 629           $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 630           if ($contactType) {
 631             if ($formatted['contact_type'] == $contactType) {
 632               //validation of subtype for update mode
 633               //CRM-5125
 634               $contactSubType = NULL;
 635               if (!empty($params['contact_sub_type'])) {
 636                 $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 637               }
 638
 639               if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 640
 641                 $message = "Mismatched contact SubTypes :";
 642                 array_unshift($values, $message);
 643                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 644               }
 645               else {
 646                 $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $params['id'], FALSE, $this->_dedupeRuleGroupID);
 647                 $this->_retCode = CRM_Import_Parser::VALID;
 648               }
 649             }
 650             else {
 651               $message = "Mismatched contact Types :";
 652               array_unshift($values, $message);
 653               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 654             }
 655           }
 656           else {
 657             // we should avoid multiple errors for single record
 658             // since we have already retCode and we trying to force again.
 659             if ($this->_retCode != CRM_Import_Parser::NO_MATCH) {
 660               $message = "No contact found for this contact ID:" . $params['id'];
 661               array_unshift($values, $message);
 662               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 663             }
 664           }
 665         }
 666         else {
 667           //CRM-4148
 668           //now we want to create new contact on update/fill also.
 669           $createNewContact = TRUE;
 670         }
 671       }
 672
 673       if (isset($newContact) && is_a($newContact, 'CRM_Contact_BAO_Contact')) {
 674         $relationship = TRUE;
 675       }
 676     }
 677
 678     //fixed CRM-4148
 679     //now we create new contact in update/fill mode also.
 680     $contactID = NULL;
 681     if ($createNewContact || ($this->_retCode != CRM_Import_Parser::NO_MATCH && $this->_updateWithId)) {
 682       // @todo - there are multiple places where formatting is done that need consolidation.
 683       // This handles where the label has been passed in and it has gotten this far.
 684       // probably a bunch of hard-coded stuff could be removed to rely on this.
 685       $fields = Contact::getFields(FALSE)
 686         ->addWhere('options', '=', TRUE)
 687         ->setLoadOptions(TRUE)
 688         ->execute()->indexBy('name');
 689       foreach ($fields as $fieldName => $fieldSpec) {
 690         if (isset($formatted[$fieldName]) && is_array($formatted[$fieldName])) {
 691           // If we have an array at this stage, it's probably a multi-select
 692           // field that has already been parsed properly into the value that
 693           // should be inserted into the database.
 694           continue;
 695         }
 696         if (!empty($formatted[$fieldName])
 697           && empty($fieldSpec['options'][$formatted[$fieldName]])) {
 698           $formatted[$fieldName] = array_search($formatted[$fieldName], $fieldSpec['options'], TRUE) ?? $formatted[$fieldName];
 699         }
 700       }
 701       //CRM-4430, don't carry if not submitted.
 702       if ($this->_updateWithId && !empty($params['id'])) {
 703         $contactID = $params['id'];
 704       }
 705       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactID, TRUE, $this->_dedupeRuleGroupID);
 706     }
 707
 708     if (isset($newContact) && is_object($newContact) && ($newContact instanceof CRM_Contact_BAO_Contact)) {
 709       $relationship = TRUE;
 710       $newContact = clone($newContact);
 711       $contactID = $newContact->id;
 712       $this->_newContacts[] = $contactID;
 713
 714       //get return code if we create new contact in update mode, CRM-4148
 715       if ($this->_updateWithId) {
 716         $this->_retCode = CRM_Import_Parser::VALID;
 717       }
 718     }
 719     elseif (isset($newContact) && CRM_Core_Error::isAPIError($newContact, CRM_Core_Error::DUPLICATE_CONTACT)) {
 720       // if duplicate, no need of further processing
 721       if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
 722         $errorMessage = "Skipping duplicate record";
 723         array_unshift($values, $errorMessage);
 724         $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', $errorMessage);
 725         return CRM_Import_Parser::DUPLICATE;
 726       }
 727
 728       $relationship = TRUE;
 729       // CRM-10433/CRM-20739 - IDs could be string or array; handle accordingly
 730       if (!is_array($dupeContactIDs = $newContact['error_message']['params'][0])) {
 731         $dupeContactIDs = explode(',', $dupeContactIDs);
 732       }
 733       $dupeCount = count($dupeContactIDs);
 734       $contactID = array_pop($dupeContactIDs);
 735       // check to see if we had more than one duplicate contact id.
 736       // if we have more than one, the record will be rejected below
 737       if ($dupeCount == 1) {
 738         // there was only one dupe, we will continue normally...
 739         if (!in_array($contactID, $this->_newContacts)) {
 740           $this->_newContacts[] = $contactID;
 741         }
 742       }
 743     }
 744
 745     if ($contactID) {
 746       // call import hook
 747       $currentImportID = end($values);
 748
 749       $hookParams = [
 750         'contactID' => $contactID,
 751         'importID' => $currentImportID,
 752         'importTempTable' => $this->_tableName,
 753         'fieldHeaders' => $this->_mapperKeys,
 754         'fields' => $this->_activeFields,
 755       ];
 756
 757       CRM_Utils_Hook::import('Contact', 'process', $this, $hookParams);
 758     }
 759
 760     if ($relationship) {
 761       $primaryContactId = NULL;
 762       if (CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 763         if ($dupeCount == 1 && CRM_Utils_Rule::integer($contactID)) {
 764           $primaryContactId = $contactID;
 765         }
 766       }
 767       else {
 768         $primaryContactId = $newContact->id;
 769       }
 770
 771       if ((CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || is_a($newContact, 'CRM_Contact_BAO_Contact')) && $primaryContactId) {
 772
 773         //relationship contact insert
 774         foreach ($params as $key => $field) {
 775           [$id, $first, $second] = CRM_Utils_System::explode('_', $key, 3);
 776           if (!($first == 'a' && $second == 'b') && !($first == 'b' && $second == 'a')) {
 777             continue;
 778           }
 779
 780           $relationType = new CRM_Contact_DAO_RelationshipType();
 781           $relationType->id = $id;
 782           $relationType->find(TRUE);
 783           $direction = "contact_sub_type_$second";
 784
 785           $formatting = [
 786             'contact_type' => $params[$key]['contact_type'],
 787           ];
 788
 789           //set subtype for related contact CRM-5125
 790           if (isset($relationType->$direction)) {
 791             //validation of related contact subtype for update mode
 792             if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $params[$key]) && $relCsType != $relationType->$direction) {
 793               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 794               array_unshift($values, $errorMessage);
 795               return CRM_Import_Parser::NO_MATCH;
 796             }
 797             else {
 798               $formatting['contact_sub_type'] = $relationType->$direction;
 799             }
 800           }
 801
 802           $contactFields = NULL;
 803           $contactFields = CRM_Contact_DAO_Contact::import();
 804
 805           //Relation on the basis of External Identifier.
 806           if (empty($params[$key]['id']) && !empty($params[$key]['external_identifier'])) {
 807             $params[$key]['id'] = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['external_identifier'], 'id', 'external_identifier');
 808           }
 809           // check for valid related contact id in update/fill mode, CRM-4424
 810           if (in_array($onDuplicate, [
 811             CRM_Import_Parser::DUPLICATE_UPDATE,
 812             CRM_Import_Parser::DUPLICATE_FILL,
 813           ]) && !empty($params[$key]['id'])) {
 814             $relatedContactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_type');
 815             if (!$relatedContactType) {
 816               $errorMessage = ts("No contact found for this related contact ID: %1", [1 => $params[$key]['id']]);
 817               array_unshift($values, $errorMessage);
 818               return CRM_Import_Parser::NO_MATCH;
 819             }
 820
 821             //validation of related contact subtype for update mode
 822             //CRM-5125
 823             $relatedCsType = NULL;
 824             if (!empty($formatting['contact_sub_type'])) {
 825               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_sub_type');
 826             }
 827
 828             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params[$key]['id'], $relatedCsType) &&
 829                 $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))
 830             ) {
 831               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.") . ' ' . ts("ID: %1", [1 => $params[$key]['id']]);
 832               array_unshift($values, $errorMessage);
 833               return CRM_Import_Parser::NO_MATCH;
 834             }
 835             // get related contact id to format data in update/fill mode,
 836             //if external identifier is present, CRM-4423
 837             $formatting['id'] = $params[$key]['id'];
 838           }
 839
 840           //format common data, CRM-4062
 841           $this->formatCommonData($field, $formatting, $contactFields);
 842
 843           //do we have enough fields to create related contact.
 844           $allowToCreate = $this->checkRelatedContactFields($key, $formatting);
 845
 846           if (!$allowToCreate) {
 847             $errorMessage = ts('Related contact required fields are missing.');
 848             array_unshift($values, $errorMessage);
 849             return CRM_Import_Parser::NO_MATCH;
 850           }
 851
 852           //fixed for CRM-4148
 853           if (!empty($params[$key]['id'])) {
 854             $contact = [
 855               'contact_id' => $params[$key]['id'],
 856             ];
 857             $defaults = [];
 858             $relatedNewContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
 859           }
 860           else {
 861             $relatedNewContact = $this->createContact($formatting, $contactFields, $onDuplicate, NULL, FALSE);
 862           }
 863
 864           if (is_object($relatedNewContact) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 865             $relatedNewContact = clone($relatedNewContact);
 866           }
 867
 868           $matchedIDs = [];
 869           // To update/fill contact, get the matching contact Ids if duplicate contact found
 870           // otherwise get contact Id from object of related contact
 871           if (is_array($relatedNewContact) && civicrm_error($relatedNewContact)) {
 872             if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 873               $matchedIDs = $relatedNewContact['error_message']['params'][0];
 874               if (!is_array($matchedIDs)) {
 875                 $matchedIDs = explode(',', $matchedIDs);
 876               }
 877             }
 878             else {
 879               $errorMessage = $relatedNewContact['error_message'];
 880               array_unshift($values, $errorMessage);
 881               $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 882               return CRM_Import_Parser::ERROR;
 883             }
 884           }
 885           else {
 886             $matchedIDs[] = $relatedNewContact->id;
 887           }
 888           // update/fill related contact after getting matching Contact Ids, CRM-4424
 889           if (in_array($onDuplicate, [
 890             CRM_Import_Parser::DUPLICATE_UPDATE,
 891             CRM_Import_Parser::DUPLICATE_FILL,
 892           ])) {
 893             //validation of related contact subtype for update mode
 894             //CRM-5125
 895             $relatedCsType = NULL;
 896             if (!empty($formatting['contact_sub_type'])) {
 897               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $matchedIDs[0], 'contact_sub_type');
 898             }
 899
 900             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($matchedIDs[0], $relatedCsType) && $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))) {
 901               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 902               array_unshift($values, $errorMessage);
 903               return CRM_Import_Parser::NO_MATCH;
 904             }
 905             else {
 906               $updatedContact = $this->createContact($formatting, $contactFields, $onDuplicate, $matchedIDs[0]);
 907             }
 908           }
 909           static $relativeContact = [];
 910           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 911             if (count($matchedIDs) >= 1) {
 912               $relContactId = $matchedIDs[0];
 913               //add relative contact to count during update & fill mode.
 914               //logic to make count distinct by contact id.
 915               if ($this->_newRelatedContacts || !empty($relativeContact)) {
 916                 $reContact = array_keys($relativeContact, $relContactId);
 917
 918                 if (empty($reContact)) {
 919                   $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 920                 }
 921               }
 922               else {
 923                 $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 924               }
 925             }
 926           }
 927           else {
 928             $relContactId = $relatedNewContact->id;
 929             $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 930           }
 931
 932           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 933             //fix for CRM-1993.Checks for duplicate related contacts
 934             if (count($matchedIDs) >= 1) {
 935               //if more than one duplicate contact
 936               //found, create relationship with first contact
 937               // now create the relationship record
 938               $relationParams = [
 939                 'relationship_type_id' => $key,
 940                 'contact_check' => [
 941                   $relContactId => 1,
 942                 ],
 943                 'is_active' => 1,
 944                 'skipRecentView' => TRUE,
 945               ];
 946
 947               // we only handle related contact success, we ignore failures for now
 948               // at some point wold be nice to have related counts as separate
 949               $relationIds = [
 950                 'contact' => $primaryContactId,
 951               ];
 952
 953               [$valid, $duplicate] = self::legacyCreateMultiple($relationParams, $relationIds);
 954
 955               if ($valid || $duplicate) {
 956                 $relationIds['contactTarget'] = $relContactId;
 957                 $action = ($duplicate) ? CRM_Core_Action::UPDATE : CRM_Core_Action::ADD;
 958                 CRM_Contact_BAO_Relationship::relatedMemberships($primaryContactId, $relationParams, $relationIds, $action);
 959               }
 960
 961               //handle current employer, CRM-3532
 962               if ($valid) {
 963                 $allRelationships = CRM_Core_PseudoConstant::relationshipType('name');
 964                 $relationshipTypeId = str_replace([
 965                   '_a_b',
 966                   '_b_a',
 967                 ], [
 968                   '',
 969                   '',
 970                 ], $key);
 971                 $relationshipType = str_replace($relationshipTypeId . '_', '', $key);
 972                 $orgId = $individualId = NULL;
 973                 if ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employee of') {
 974                   $orgId = $relContactId;
 975                   $individualId = $primaryContactId;
 976                 }
 977                 elseif ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employer of') {
 978                   $orgId = $primaryContactId;
 979                   $individualId = $relContactId;
 980                 }
 981                 if ($orgId && $individualId) {
 982                   $currentEmpParams[$individualId] = $orgId;
 983                   CRM_Contact_BAO_Contact_Utils::setCurrentEmployer($currentEmpParams);
 984                 }
 985               }
 986             }
 987           }
 988         }
 989       }
 990     }
 991     if ($this->_updateWithId) {
 992       //return warning if street address is unparsed, CRM-5886
 993       return $this->processMessage($values, $statusFieldName, $this->_retCode);
 994     }
 995     //dupe checking
 996     if (is_array($newContact) && civicrm_error($newContact)) {
 997       $code = NULL;
 998
 999       if (($code = CRM_Utils_Array::value('code', $newContact['error_message'])) && ($code == CRM_Core_Error::DUPLICATE_CONTACT)) {
1000         return $this->handleDuplicateError($newContact, $statusFieldName, $values, $onDuplicate, $formatted, $contactFields);
1001       }
1002       // Not a dupe, so we had an error
1003       $errorMessage = $newContact['error_message'];
1004       array_unshift($values, $errorMessage);
1005       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
1006       return CRM_Import_Parser::ERROR;
1007
1008     }
1009     // sleep(3);
1010     return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID);
1011   }
1012
1013   /**
1014    * Only called from import now... plus one place outside of core & tests.
1015    *
1016    * @todo - deprecate more aggressively - will involve copying to the import
1017    * class, adding a deprecation notice here & removing from tests.
1018    *
1019    * Takes an associative array and creates a relationship object.
1020    *
1021    * @deprecated For single creates use the api instead (it's tested).
1022    * For multiple a new variant of this function needs to be written and migrated to as this is a bit
1023    * nasty
1024    *
1025    * @param array $params
1026    *   (reference ) an assoc array of name/value pairs.
1027    * @param array $ids
1028    *   The array that holds all the db ids.
1029    *   per http://wiki.civicrm.org/confluence/display/CRM/Database+layer
1030    *  "we are moving away from the $ids param "
1031    *
1032    * @return array
1033    * @throws \CRM_Core_Exception
1034    */
1035   private static function legacyCreateMultiple($params, $ids = []) {
1036     // clarify that the only key ever pass in the ids array is 'contact'
1037     // There is legacy handling for other keys but a universe search on
1038     // calls to this function (not supported to be called from outside core)
1039     // only returns 2 calls - one in CRM_Contact_Import_Parser_Contact
1040     // and the other in jma grant applications (CRM_Grant_Form_Grant_Confirm)
1041     // both only pass in contact as a key here.
1042     $contactID = $ids['contact'];
1043     unset($ids);
1044     // There is only ever one value passed in from the 2 places above that call
1045     // this - by clarifying here like this we can cleanup within this
1046     // function without having to do more universe searches.
1047     $relatedContactID = key($params['contact_check']);
1048
1049     // check if the relationship is valid between contacts.
1050     // step 1: check if the relationship is valid if not valid skip and keep the count
1051     // step 2: check the if two contacts already have a relationship if yes skip and keep the count
1052     // step 3: if valid relationship then add the relation and keep the count
1053
1054     // step 1
1055     [$contactFields['relationship_type_id'], $firstLetter, $secondLetter] = explode('_', $params['relationship_type_id']);
1056     $contactFields['contact_id_' . $firstLetter] = $contactID;
1057     $contactFields['contact_id_' . $secondLetter] = $relatedContactID;
1058     if (!CRM_Contact_BAO_Relationship::checkRelationshipType($contactFields['contact_id_a'], $contactFields['contact_id_b'],
1059       $contactFields['relationship_type_id'])) {
1060       return [0, 0];
1061     }
1062
1063     if (
1064       CRM_Contact_BAO_Relationship::checkDuplicateRelationship(
1065         $contactFields,
1066         $contactID,
1067         // step 2
1068         $relatedContactID
1069       )
1070     ) {
1071       return [0, 1];
1072     }
1073
1074     $singleInstanceParams = array_merge($params, $contactFields);
1075     CRM_Contact_BAO_Relationship::add($singleInstanceParams);
1076     return [1, 0];
1077   }
1078
1079   /**
1080    * Format common params data to proper format to store.
1081    *
1082    * @param array $params
1083    *   Contain record values.
1084    * @param array $formatted
1085    *   Array of formatted data.
1086    * @param array $contactFields
1087    *   Contact DAO fields.
1088    */
1089   private function formatCommonData($params, &$formatted, $contactFields) {
1090     $csType = [
1091       CRM_Utils_Array::value('contact_type', $formatted),
1092     ];
1093
1094     //CRM-5125
1095     //add custom fields for contact sub type
1096     if (!empty($this->_contactSubType)) {
1097       $csType = $this->_contactSubType;
1098     }
1099
1100     if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $formatted)) {
1101       $csType = $relCsType;
1102     }
1103
1104     $customFields = CRM_Core_BAO_CustomField::getFields($formatted['contact_type'], FALSE, FALSE, $csType);
1105
1106     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
1107     $customFields = $customFields + $addressCustomFields;
1108
1109     //if a Custom Email Greeting, Custom Postal Greeting or Custom Addressee is mapped, and no "Greeting / Addressee Type ID" is provided, then automatically set the type = Customized, CRM-4575
1110     $elements = [
1111       'email_greeting_custom' => 'email_greeting',
1112       'postal_greeting_custom' => 'postal_greeting',
1113       'addressee_custom' => 'addressee',
1114     ];
1115     foreach ($elements as $k => $v) {
1116       if (array_key_exists($k, $params) && !(array_key_exists($v, $params))) {
1117         $label = key(CRM_Core_OptionGroup::values($v, TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1118         $params[$v] = $label;
1119       }
1120     }
1121
1122     //format date first
1123     $session = CRM_Core_Session::singleton();
1124     $dateType = $session->get("dateTypes");
1125     foreach ($params as $key => $val) {
1126       $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1127       if ($customFieldID &&
1128         !array_key_exists($customFieldID, $addressCustomFields)
1129       ) {
1130         //we should not update Date to null, CRM-4062
1131         if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
1132           //CRM-21267
1133           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
1134         }
1135         elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
1136           if (empty($val) && !is_numeric($val) && $this->_onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) {
1137             //retain earlier value when Import mode is `Fill`
1138             unset($params[$key]);
1139           }
1140           else {
1141             $params[$key] = CRM_Utils_String::strtoboolstr($val);
1142           }
1143         }
1144       }
1145
1146       if ($key == 'birth_date' && $val) {
1147         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
1148       }
1149       elseif ($key == 'deceased_date' && $val) {
1150         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
1151         $params['is_deceased'] = 1;
1152       }
1153       elseif ($key == 'is_deceased' && $val) {
1154         $params[$key] = CRM_Utils_String::strtoboolstr($val);
1155       }
1156     }
1157
1158     //now format custom data.
1159     foreach ($params as $key => $field) {
1160       if (is_array($field)) {
1161         $isAddressCustomField = FALSE;
1162         foreach ($field as $value) {
1163           $break = FALSE;
1164           if (is_array($value)) {
1165             foreach ($value as $name => $testForEmpty) {
1166               if ($addressCustomFieldID = CRM_Core_BAO_CustomField::getKeyID($name)) {
1167                 $isAddressCustomField = TRUE;
1168                 break;
1169               }
1170               // check if $value does not contain IM provider or phoneType
1171               if (($name !== 'phone_type_id' || $name !== 'provider_id') && ($testForEmpty === '' || $testForEmpty == NULL)) {
1172                 $break = TRUE;
1173                 break;
1174               }
1175             }
1176           }
1177           else {
1178             $break = TRUE;
1179           }
1180
1181           if (!$break) {
1182             if (!empty($value['location_type_id'])) {
1183               $this->formatLocationBlock($value, $formatted);
1184             }
1185             else {
1186               // @todo - this is still reachable - e.g. import with related contact info like firstname,lastname,spouse-first-name,spouse-last-name,spouse-home-phone
1187               CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
1188               $this->formatContactParameters($value, $formatted);
1189             }
1190           }
1191         }
1192         if (!$isAddressCustomField) {
1193           continue;
1194         }
1195       }
1196
1197       $formatValues = [
1198         $key => $field,
1199       ];
1200
1201       if (($key !== 'preferred_communication_method') && (array_key_exists($key, $contactFields))) {
1202         // due to merging of individual table and
1203         // contact table, we need to avoid
1204         // preferred_communication_method forcefully
1205         $formatValues['contact_type'] = $formatted['contact_type'];
1206       }
1207
1208       if ($key == 'id' && isset($field)) {
1209         $formatted[$key] = $field;
1210       }
1211       $this->formatContactParameters($formatValues, $formatted);
1212
1213       //Handling Custom Data
1214       // note: Address custom fields will be handled separately inside formatContactParameters
1215       if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
1216         array_key_exists($customFieldID, $customFields) &&
1217         !array_key_exists($customFieldID, $addressCustomFields)
1218       ) {
1219
1220         $extends = $customFields[$customFieldID]['extends'] ?? NULL;
1221         $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1222         $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
1223         $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1224
1225         if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
1226           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1227           foreach ($customOption as $customValue) {
1228             $val = $customValue['value'] ?? NULL;
1229             $label = strtolower($customValue['label'] ?? '');
1230             $value = strtolower(trim($formatted[$key]));
1231             if (($value == $label) || ($value == strtolower($val))) {
1232               $params[$key] = $formatted[$key] = $val;
1233             }
1234           }
1235         }
1236         elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
1237           $mulValues = explode(',', $formatted[$key]);
1238           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1239           $formatted[$key] = [];
1240           $params[$key] = [];
1241           foreach ($mulValues as $v1) {
1242             foreach ($customOption as $v2) {
1243               if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1244                 (strtolower($v2['value']) == strtolower(trim($v1)))
1245               ) {
1246                 if ($htmlType == 'CheckBox') {
1247                   $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
1248                 }
1249                 else {
1250                   $params[$key][] = $formatted[$key][] = $v2['value'];
1251                 }
1252               }
1253             }
1254           }
1255         }
1256       }
1257     }
1258
1259     if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields) &&
1260       !array_key_exists($customFieldID, $addressCustomFields)
1261     ) {
1262       // @todo calling api functions directly is not supported
1263       _civicrm_api3_custom_format_params($params, $formatted, $extends);
1264     }
1265
1266     // to check if not update mode and unset the fields with empty value.
1267     if (!$this->_updateWithId && array_key_exists('custom', $formatted)) {
1268       foreach ($formatted['custom'] as $customKey => $customvalue) {
1269         if (empty($formatted['custom'][$customKey][-1]['is_required'])) {
1270           $formatted['custom'][$customKey][-1]['is_required'] = $customFields[$customKey]['is_required'];
1271         }
1272         $emptyValue = $customvalue[-1]['value'] ?? NULL;
1273         if (!isset($emptyValue)) {
1274           unset($formatted['custom'][$customKey]);
1275         }
1276       }
1277     }
1278
1279     // parse street address, CRM-5450
1280     if ($this->_parseStreetAddress) {
1281       if (array_key_exists('address', $formatted) && is_array($formatted['address'])) {
1282         foreach ($formatted['address'] as $instance => & $address) {
1283           $streetAddress = $address['street_address'] ?? NULL;
1284           if (empty($streetAddress)) {
1285             continue;
1286           }
1287           // parse address field.
1288           $parsedFields = CRM_Core_BAO_Address::parseStreetAddress($streetAddress);
1289
1290           //street address consider to be parsed properly,
1291           //If we get street_name and street_number.
1292           if (empty($parsedFields['street_name']) || empty($parsedFields['street_number'])) {
1293             $parsedFields = array_fill_keys(array_keys($parsedFields), '');
1294           }
1295
1296           // merge parse address w/ main address block.
1297           $address = array_merge($address, $parsedFields);
1298         }
1299       }
1300     }
1301   }
1302
1303   /**
1304    * Get the array of successfully imported contact id's
1305    *
1306    * @return array
1307    */
1308   public function getImportedContacts() {
1309     return $this->_newContacts;
1310   }
1311
1312   /**
1313    * Get the array of successfully imported related contact id's
1314    *
1315    * @return array
1316    */
1317   public function &getRelatedImportedContacts() {
1318     return $this->_newRelatedContacts;
1319   }
1320
1321   /**
1322    * Check if an error in custom data.
1323    *
1324    * @param array $params
1325    * @param string $errorMessage
1326    *   A string containing all the error-fields.
1327    *
1328    * @param null $csType
1329    * @param null $relationships
1330    */
1331   public static function isErrorInCustomData($params, &$errorMessage, $csType = NULL, $relationships = NULL) {
1332     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1333     $errors = [];
1334
1335     if (!empty($params['contact_sub_type'])) {
1336       $csType = $params['contact_sub_type'] ?? NULL;
1337     }
1338
1339     if (empty($params['contact_type'])) {
1340       $params['contact_type'] = 'Individual';
1341     }
1342
1343     // get array of subtypes - CRM-18708
1344     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1345       $csType = self::getSubtypes($params['contact_type']);
1346     }
1347
1348     if (is_array($csType)) {
1349       // fetch custom fields for every subtype and add it to $customFields array
1350       // CRM-18708
1351       $customFields = [];
1352       foreach ($csType as $cType) {
1353         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1354       }
1355     }
1356     else {
1357       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1358     }
1359
1360     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
1361     $customFields = $customFields + $addressCustomFields;
1362     foreach ($params as $key => $value) {
1363       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1364         /* check if it's a valid custom field id */
1365
1366         if (!array_key_exists($customFieldID, $customFields)) {
1367           $errors[] = ts('field ID');
1368         }
1369         // validate null values for required custom fields of type boolean
1370         if (!empty($customFields[$customFieldID]['is_required']) && (empty($params['custom_' . $customFieldID]) && !is_numeric($params['custom_' . $customFieldID])) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
1371           $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1372         }
1373
1374         //For address custom fields, we do get actual custom field value as an inner array of
1375         //values so need to modify
1376         if (array_key_exists($customFieldID, $addressCustomFields)) {
1377           $value = $value[0][$key];
1378         }
1379         /* validate the data against the CF type */
1380
1381         if ($value) {
1382           $dataType = $customFields[$customFieldID]['data_type'];
1383           $htmlType = $customFields[$customFieldID]['html_type'];
1384           $isSerialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1385           if ($dataType == 'Date') {
1386             if (array_key_exists($customFieldID, $addressCustomFields) && CRM_Utils_Date::convertToDefaultDate($params[$key][0], $dateType, $key)) {
1387               $value = $params[$key][0][$key];
1388             }
1389             elseif (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1390               $value = $params[$key];
1391             }
1392             else {
1393               $errors[] = $customFields[$customFieldID]['label'];
1394             }
1395           }
1396           elseif ($dataType == 'Boolean') {
1397             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1398               $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1399             }
1400           }
1401           // need not check for label filed import
1402           $selectHtmlTypes = [
1403             'CheckBox',
1404             'Select',
1405             'Radio',
1406           ];
1407           if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1408             $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1409             if (!$valid) {
1410               $errors[] = $customFields[$customFieldID]['label'];
1411             }
1412           }
1413
1414           // check for values for custom fields for checkboxes and multiselect
1415           if ($isSerialized && $dataType != 'ContactReference') {
1416             $value = trim($value);
1417             $value = str_replace('|', ',', $value);
1418             $mulValues = explode(',', $value);
1419             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1420             foreach ($mulValues as $v1) {
1421               if (strlen($v1) == 0) {
1422                 continue;
1423               }
1424
1425               $flag = FALSE;
1426               foreach ($customOption as $v2) {
1427                 if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1428                   $flag = TRUE;
1429                 }
1430               }
1431
1432               if (!$flag) {
1433                 $errors[] = $customFields[$customFieldID]['label'];
1434               }
1435             }
1436           }
1437           elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1438             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1439             $flag = FALSE;
1440             foreach ($customOption as $v2) {
1441               if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1442                 $flag = TRUE;
1443               }
1444             }
1445             if (!$flag) {
1446               $errors[] = $customFields[$customFieldID]['label'];
1447             }
1448           }
1449           elseif ($isSerialized && $dataType === 'StateProvince') {
1450             $mulValues = explode(',', $value);
1451             foreach ($mulValues as $stateValue) {
1452               if ($stateValue) {
1453                 if (self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvinceAbbreviation()) || self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvince())) {
1454                   continue;
1455                 }
1456                 else {
1457                   $errors[] = $customFields[$customFieldID]['label'];
1458                 }
1459               }
1460             }
1461           }
1462           elseif ($isSerialized && $dataType == 'Country') {
1463             $mulValues = explode(',', $value);
1464             foreach ($mulValues as $countryValue) {
1465               if ($countryValue) {
1466                 CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1467                 CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1468                 $limitCodes = CRM_Core_BAO_Country::countryLimit();
1469
1470                 $error = TRUE;
1471                 foreach ([
1472                   $countryNames,
1473                   $countryIsoCodes,
1474                   $limitCodes,
1475                 ] as $values) {
1476                   if (in_array(trim($countryValue), $values)) {
1477                     $error = FALSE;
1478                     break;
1479                   }
1480                 }
1481
1482                 if ($error) {
1483                   $errors[] = $customFields[$customFieldID]['label'];
1484                 }
1485               }
1486             }
1487           }
1488         }
1489       }
1490       elseif (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1491         //CRM-5125
1492         //supporting custom data of related contact subtypes
1493         $relation = NULL;
1494         if ($relationships) {
1495           if (array_key_exists($key, $relationships)) {
1496             $relation = $key;
1497           }
1498           elseif (CRM_Utils_Array::key($key, $relationships)) {
1499             $relation = CRM_Utils_Array::key($key, $relationships);
1500           }
1501         }
1502         if (!empty($relation)) {
1503           [$id, $first, $second] = CRM_Utils_System::explode('_', $relation, 3);
1504           $direction = "contact_sub_type_$second";
1505           $relationshipType = new CRM_Contact_BAO_RelationshipType();
1506           $relationshipType->id = $id;
1507           if ($relationshipType->find(TRUE)) {
1508             if (isset($relationshipType->$direction)) {
1509               $params[$key]['contact_sub_type'] = $relationshipType->$direction;
1510             }
1511           }
1512         }
1513
1514         self::isErrorInCustomData($params[$key], $errorMessage, $csType, $relationships);
1515       }
1516     }
1517     if ($errors) {
1518       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1519     }
1520   }
1521
1522   /**
1523    * Check if value present in all genders or.
1524    * as a substring of any gender value, if yes than return corresponding gender.
1525    * eg value might be  m/M, ma/MA, mal/MAL, male return 'Male'
1526    * but if value is 'maleabc' than return false
1527    *
1528    * @param string $gender
1529    *   Check this value across gender values.
1530    *
1531    * retunr gender value / false
1532    *
1533    * @return bool
1534    */
1535   public function checkGender($gender) {
1536     $gender = trim($gender, '.');
1537     if (!$gender) {
1538       return FALSE;
1539     }
1540
1541     $allGenders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
1542     foreach ($allGenders as $key => $value) {
1543       if (strlen($gender) > strlen($value)) {
1544         continue;
1545       }
1546       if ($gender == $value) {
1547         return $value;
1548       }
1549       if (substr_compare($value, $gender, 0, strlen($gender), TRUE) === 0) {
1550         return $value;
1551       }
1552     }
1553
1554     return FALSE;
1555   }
1556
1557   /**
1558    * Check if an error in Core( non-custom fields ) field
1559    *
1560    * @param array $params
1561    * @param string $errorMessage
1562    *   A string containing all the error-fields.
1563    */
1564   public function isErrorInCoreData($params, &$errorMessage) {
1565     $errors = [];
1566     foreach ($params as $key => $value) {
1567       if ($value) {
1568         $session = CRM_Core_Session::singleton();
1569         $dateType = $session->get("dateTypes");
1570
1571         switch ($key) {
1572           case 'birth_date':
1573             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1574               if (!CRM_Utils_Rule::date($params[$key])) {
1575                 $errors[] = ts('Birth Date');
1576               }
1577             }
1578             else {
1579               $errors[] = ts('Birth-Date');
1580             }
1581             break;
1582
1583           case 'deceased_date':
1584             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1585               if (!CRM_Utils_Rule::date($params[$key])) {
1586                 $errors[] = ts('Deceased Date');
1587               }
1588             }
1589             else {
1590               $errors[] = ts('Deceased Date');
1591             }
1592             break;
1593
1594           case 'is_deceased':
1595             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1596               $errors[] = ts('Deceased');
1597             }
1598             break;
1599
1600           case 'gender_id':
1601             if (!self::checkGender($value)) {
1602               $errors[] = ts('Gender');
1603             }
1604             break;
1605
1606           case 'preferred_communication_method':
1607             $preffComm = [];
1608             $preffComm = explode(',', $value);
1609             foreach ($preffComm as $v) {
1610               if (!self::in_value(trim($v), CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method'))) {
1611                 $errors[] = ts('Preferred Communication Method');
1612               }
1613             }
1614             break;
1615
1616           case 'preferred_mail_format':
1617             if (!array_key_exists(strtolower($value), array_change_key_case(CRM_Core_SelectValues::pmf(), CASE_LOWER))) {
1618               $errors[] = ts('Preferred Mail Format');
1619             }
1620             break;
1621
1622           case 'individual_prefix':
1623           case 'prefix_id':
1624             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id'))) {
1625               $errors[] = ts('Individual Prefix');
1626             }
1627             break;
1628
1629           case 'individual_suffix':
1630           case 'suffix_id':
1631             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id'))) {
1632               $errors[] = ts('Individual Suffix');
1633             }
1634             break;
1635
1636           case 'state_province':
1637             if (!empty($value)) {
1638               foreach ($value as $stateValue) {
1639                 if ($stateValue['state_province']) {
1640                   if (self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvinceAbbreviation()) ||
1641                     self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvince())
1642                   ) {
1643                     continue;
1644                   }
1645                   else {
1646                     $errors[] = ts('State/Province');
1647                   }
1648                 }
1649               }
1650             }
1651             break;
1652
1653           case 'country':
1654             if (!empty($value)) {
1655               foreach ($value as $stateValue) {
1656                 if ($stateValue['country']) {
1657                   CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1658                   CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1659                   $limitCodes = CRM_Core_BAO_Country::countryLimit();
1660                   //If no country is selected in
1661                   //localization then take all countries
1662                   if (empty($limitCodes)) {
1663                     $limitCodes = $countryIsoCodes;
1664                   }
1665
1666                   if (self::in_value($stateValue['country'], $limitCodes) || self::in_value($stateValue['country'], CRM_Core_PseudoConstant::country())) {
1667                     continue;
1668                   }
1669                   if (self::in_value($stateValue['country'], $countryIsoCodes) || self::in_value($stateValue['country'], $countryNames)) {
1670                     $errors[] = ts('Country input value is in table but not "available": "This Country is valid but is NOT in the list of Available Countries currently configured for your site. This can be viewed and modifed from Administer > Localization > Languages Currency Locations." ');
1671                   }
1672                   else {
1673                     $errors[] = ts('Country input value not in country table: "The Country value appears to be invalid. It does not match any value in CiviCRM table of countries."');
1674                   }
1675                 }
1676               }
1677             }
1678             break;
1679
1680           case 'county':
1681             if (!empty($value)) {
1682               foreach ($value as $county) {
1683                 if ($county['county']) {
1684                   $countyNames = CRM_Core_PseudoConstant::county();
1685                   if (!empty($county['county']) && !in_array($county['county'], $countyNames)) {
1686                     $errors[] = ts('County input value not in county table: The County value appears to be invalid. It does not match any value in CiviCRM table of counties.');
1687                   }
1688                 }
1689               }
1690             }
1691             break;
1692
1693           case 'geo_code_1':
1694             if (!empty($value)) {
1695               foreach ($value as $codeValue) {
1696                 if (!empty($codeValue['geo_code_1'])) {
1697                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_1'])) {
1698                     continue;
1699                   }
1700                   $errors[] = ts('Geo code 1');
1701                 }
1702               }
1703             }
1704             break;
1705
1706           case 'geo_code_2':
1707             if (!empty($value)) {
1708               foreach ($value as $codeValue) {
1709                 if (!empty($codeValue['geo_code_2'])) {
1710                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_2'])) {
1711                     continue;
1712                   }
1713                   $errors[] = ts('Geo code 2');
1714                 }
1715               }
1716             }
1717             break;
1718
1719           //check for any error in email/postal greeting, addressee,
1720           //custom email/postal greeting, custom addressee, CRM-4575
1721
1722           case 'email_greeting':
1723             $emailGreetingFilter = [
1724               'contact_type' => $this->_contactType,
1725               'greeting_type' => 'email_greeting',
1726             ];
1727             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($emailGreetingFilter))) {
1728               $errors[] = ts('Email Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Email Greetings for valid values');
1729             }
1730             break;
1731
1732           case 'postal_greeting':
1733             $postalGreetingFilter = [
1734               'contact_type' => $this->_contactType,
1735               'greeting_type' => 'postal_greeting',
1736             ];
1737             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($postalGreetingFilter))) {
1738               $errors[] = ts('Postal Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Postal Greetings for valid values');
1739             }
1740             break;
1741
1742           case 'addressee':
1743             $addresseeFilter = [
1744               'contact_type' => $this->_contactType,
1745               'greeting_type' => 'addressee',
1746             ];
1747             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($addresseeFilter))) {
1748               $errors[] = ts('Addressee must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Addressee for valid values');
1749             }
1750             break;
1751
1752           case 'email_greeting_custom':
1753             if (array_key_exists('email_greeting', $params)) {
1754               $emailGreetingLabel = key(CRM_Core_OptionGroup::values('email_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1755               if (CRM_Utils_Array::value('email_greeting', $params) != $emailGreetingLabel) {
1756                 $errors[] = ts('Email Greeting - Custom');
1757               }
1758             }
1759             break;
1760
1761           case 'postal_greeting_custom':
1762             if (array_key_exists('postal_greeting', $params)) {
1763               $postalGreetingLabel = key(CRM_Core_OptionGroup::values('postal_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1764               if (CRM_Utils_Array::value('postal_greeting', $params) != $postalGreetingLabel) {
1765                 $errors[] = ts('Postal Greeting - Custom');
1766               }
1767             }
1768             break;
1769
1770           case 'addressee_custom':
1771             if (array_key_exists('addressee', $params)) {
1772               $addresseeLabel = key(CRM_Core_OptionGroup::values('addressee', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1773               if (CRM_Utils_Array::value('addressee', $params) != $addresseeLabel) {
1774                 $errors[] = ts('Addressee - Custom');
1775               }
1776             }
1777             break;
1778
1779           case 'url':
1780             if (is_array($value)) {
1781               foreach ($value as $values) {
1782                 if (!empty($values['url']) && !CRM_Utils_Rule::url($values['url'])) {
1783                   $errors[] = ts('Website');
1784                   break;
1785                 }
1786               }
1787             }
1788             break;
1789
1790           case 'do_not_email':
1791           case 'do_not_phone':
1792           case 'do_not_mail':
1793           case 'do_not_sms':
1794           case 'do_not_trade':
1795             if (CRM_Utils_Rule::boolean($value) == FALSE) {
1796               $key = ucwords(str_replace("_", " ", $key));
1797               $errors[] = $key;
1798             }
1799             break;
1800
1801           case 'email':
1802             if (is_array($value)) {
1803               foreach ($value as $values) {
1804                 if (!empty($values['email']) && !CRM_Utils_Rule::email($values['email'])) {
1805                   $errors[] = $key;
1806                   break;
1807                 }
1808               }
1809             }
1810             break;
1811
1812           default:
1813             if (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1814               //check for any relationship data ,FIX ME
1815               self::isErrorInCoreData($params[$key], $errorMessage);
1816             }
1817         }
1818       }
1819     }
1820     if ($errors) {
1821       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1822     }
1823   }
1824
1825   /**
1826    * Ckeck a value present or not in a array.
1827    *
1828    * @param $value
1829    * @param $valueArray
1830    *
1831    * @return bool
1832    */
1833   public static function in_value($value, $valueArray) {
1834     foreach ($valueArray as $key => $v) {
1835       //fix for CRM-1514
1836       if (strtolower(trim($v, ".")) == strtolower(trim($value, "."))) {
1837         return TRUE;
1838       }
1839     }
1840     return FALSE;
1841   }
1842
1843   /**
1844    * Build error-message containing error-fields
1845    *
1846    * Once upon a time there was a dev who hadn't heard of implode. That dev wrote this function.
1847    *
1848    * @todo just say no!
1849    *
1850    * @param string $errorName
1851    *   A string containing error-field name.
1852    * @param string $errorMessage
1853    *   A string containing all the error-fields, where the new errorName is concatenated.
1854    *
1855    */
1856   public static function addToErrorMsg($errorName, &$errorMessage) {
1857     if ($errorMessage) {
1858       $errorMessage .= "; $errorName";
1859     }
1860     else {
1861       $errorMessage = $errorName;
1862     }
1863   }
1864
1865   /**
1866    * Method for creating contact.
1867    *
1868    * @param array $formatted
1869    * @param array $contactFields
1870    * @param int $onDuplicate
1871    * @param int $contactId
1872    * @param bool $requiredCheck
1873    * @param int $dedupeRuleGroupID
1874    *
1875    * @return array|bool|\CRM_Contact_BAO_Contact|\CRM_Core_Error|null
1876    */
1877   public function createContact(&$formatted, &$contactFields, $onDuplicate, $contactId = NULL, $requiredCheck = TRUE, $dedupeRuleGroupID = NULL) {
1878     $dupeCheck = FALSE;
1879     $newContact = NULL;
1880
1881     if (is_null($contactId) && ($onDuplicate != CRM_Import_Parser::DUPLICATE_NOCHECK)) {
1882       $dupeCheck = (bool) ($onDuplicate);
1883     }
1884
1885     //get the prefix id etc if exists
1886     CRM_Contact_BAO_Contact::resolveDefaults($formatted, TRUE);
1887
1888     //@todo direct call to API function not supported.
1889     // setting required check to false, CRM-2839
1890     // plus we do our own required check in import
1891     try {
1892       $error = $this->deprecated_contact_check_params($formatted, $dupeCheck, $dedupeRuleGroupID);
1893       if ($error) {
1894         return $error;
1895       }
1896       $this->deprecated_validate_formatted_contact($formatted);
1897     }
1898     catch (CRM_Core_Exception $e) {
1899       return ['error_message' => $e->getMessage(), 'is_error' => 1, 'code' => $e->getCode()];
1900     }
1901
1902     if ($contactId) {
1903       $this->formatParams($formatted, $onDuplicate, (int) $contactId);
1904     }
1905
1906     // Resetting and rebuilding cache could be expensive.
1907     CRM_Core_Config::setPermitCacheFlushMode(FALSE);
1908
1909     // If a user has logged in, or accessed via a checksum
1910     // Then deliberately 'blanking' a value in the profile should remove it from their record
1911     // @todo this should either be TRUE or FALSE in the context of import - once
1912     // we figure out which we can remove all the rest.
1913     // Also note the meaning of this parameter is less than it used to
1914     // be following block cleanup.
1915     $formatted['updateBlankLocInfo'] = TRUE;
1916     if ((CRM_Core_Session::singleton()->get('authSrc') & (CRM_Core_Permission::AUTH_SRC_CHECKSUM + CRM_Core_Permission::AUTH_SRC_LOGIN)) == 0) {
1917       $formatted['updateBlankLocInfo'] = FALSE;
1918     }
1919
1920     [$data, $contactDetails] = CRM_Contact_BAO_Contact::formatProfileContactParams($formatted, $contactFields, $contactId, NULL, $formatted['contact_type']);
1921
1922     // manage is_opt_out
1923     if (array_key_exists('is_opt_out', $contactFields) && array_key_exists('is_opt_out', $formatted)) {
1924       $wasOptOut = $contactDetails['is_opt_out'] ?? FALSE;
1925       $isOptOut = $formatted['is_opt_out'];
1926       $data['is_opt_out'] = $isOptOut;
1927       // on change, create new civicrm_subscription_history entry
1928       if (($wasOptOut != $isOptOut) && !empty($contactDetails['contact_id'])) {
1929         $shParams = [
1930           'contact_id' => $contactDetails['contact_id'],
1931           'status' => $isOptOut ? 'Removed' : 'Added',
1932           'method' => 'Web',
1933         ];
1934         CRM_Contact_BAO_SubscriptionHistory::create($shParams);
1935       }
1936     }
1937
1938     $contact = civicrm_api3('Contact', 'create', $data);
1939     $cid = $contact['id'];
1940
1941     CRM_Core_Config::setPermitCacheFlushMode(TRUE);
1942
1943     $contact = [
1944       'contact_id' => $cid,
1945     ];
1946
1947     $defaults = [];
1948     $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
1949
1950     //get the id of the contact whose street address is not parsable, CRM-5886
1951     if ($this->_parseStreetAddress && is_object($newContact) && property_exists($newContact, 'address') && $newContact->address) {
1952       foreach ($newContact->address as $address) {
1953         if (!empty($address['street_address']) && (empty($address['street_number']) || empty($address['street_name']))) {
1954           $this->_unparsedStreetAddressContacts[] = [
1955             'id' => $newContact->id,
1956             'streetAddress' => $address['street_address'],
1957           ];
1958         }
1959       }
1960     }
1961     return $newContact;
1962   }
1963
1964   /**
1965    * Format params for update and fill mode.
1966    *
1967    * @param array $params
1968    *   reference to an array containing all the.
1969    *   values for import
1970    * @param int $onDuplicate
1971    * @param int $cid
1972    *   contact id.
1973    */
1974   public function formatParams(&$params, $onDuplicate, $cid) {
1975     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
1976       return;
1977     }
1978
1979     $contactParams = [
1980       'contact_id' => $cid,
1981     ];
1982
1983     $defaults = [];
1984     $contactObj = CRM_Contact_BAO_Contact::retrieve($contactParams, $defaults);
1985
1986     $modeFill = ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL);
1987
1988     $groupTree = CRM_Core_BAO_CustomGroup::getTree($params['contact_type'], NULL, $cid, 0, NULL);
1989     CRM_Core_BAO_CustomGroup::setDefaults($groupTree, $defaults, FALSE, FALSE);
1990
1991     $locationFields = [
1992       'email' => 'email',
1993       'phone' => 'phone',
1994       'im' => 'name',
1995       'website' => 'website',
1996       'address' => 'address',
1997     ];
1998
1999     $contact = get_object_vars($contactObj);
2000
2001     foreach ($params as $key => $value) {
2002       if ($key == 'id' || $key == 'contact_type') {
2003         continue;
2004       }
2005
2006       if (array_key_exists($key, $locationFields)) {
2007         continue;
2008       }
2009       if (in_array($key, [
2010         'email_greeting',
2011         'postal_greeting',
2012         'addressee',
2013       ])) {
2014         // CRM-4575, need to null custom
2015         if ($params["{$key}_id"] != 4) {
2016           $params["{$key}_custom"] = 'null';
2017         }
2018         unset($params[$key]);
2019       }
2020       else {
2021         if ($customFieldId = CRM_Core_BAO_CustomField::getKeyID($key)) {
2022           $custom_params = ['id' => $contact['id'], 'return' => $key];
2023           $getValue = civicrm_api3('Contact', 'getvalue', $custom_params);
2024           if (empty($getValue)) {
2025             unset($getValue);
2026           }
2027         }
2028         else {
2029           $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $key);
2030         }
2031         if ($key == 'contact_source') {
2032           $params['source'] = $params[$key];
2033           unset($params[$key]);
2034         }
2035
2036         if ($modeFill && isset($getValue)) {
2037           unset($params[$key]);
2038           if ($customFieldId) {
2039             // Extra values must be unset to ensure the values are not
2040             // imported.
2041             unset($params['custom'][$customFieldId]);
2042           }
2043         }
2044       }
2045     }
2046
2047     foreach ($locationFields as $locKeys) {
2048       if (isset($params[$locKeys]) && is_array($params[$locKeys])) {
2049         foreach ($params[$locKeys] as $key => $value) {
2050           if ($modeFill) {
2051             $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $locKeys);
2052
2053             if (isset($getValue)) {
2054               foreach ($getValue as $cnt => $values) {
2055                 if ($locKeys == 'website') {
2056                   if (($getValue[$cnt]['website_type_id'] == $params[$locKeys][$key]['website_type_id'])) {
2057                     unset($params[$locKeys][$key]);
2058                   }
2059                 }
2060                 else {
2061                   if ((!empty($getValue[$cnt]['location_type_id']) && !empty($params[$locKeys][$key]['location_type_id'])) && $getValue[$cnt]['location_type_id'] == $params[$locKeys][$key]['location_type_id']) {
2062                     unset($params[$locKeys][$key]);
2063                   }
2064                 }
2065               }
2066             }
2067           }
2068         }
2069         if (count($params[$locKeys]) == 0) {
2070           unset($params[$locKeys]);
2071         }
2072       }
2073     }
2074   }
2075
2076   /**
2077    * Convert any given date string to default date array.
2078    *
2079    * @param array $params
2080    *   Has given date-format.
2081    * @param array $formatted
2082    *   Store formatted date in this array.
2083    * @param int $dateType
2084    *   Type of date.
2085    * @param string $dateParam
2086    *   Index of params.
2087    */
2088   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2089     //fix for CRM-2687
2090     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
2091     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
2092   }
2093
2094   /**
2095    * Generate status and error message for unparsed street address records.
2096    *
2097    * @param array $values
2098    *   The array of values belonging to each row.
2099    * @param array $statusFieldName
2100    *   Store formatted date in this array.
2101    * @param $returnCode
2102    *
2103    * @return int
2104    */
2105   public function processMessage(&$values, $statusFieldName, $returnCode) {
2106     if (empty($this->_unparsedStreetAddressContacts)) {
2107       $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '');
2108     }
2109     else {
2110       $errorMessage = ts("Record imported successfully but unable to parse the street address: ");
2111       foreach ($this->_unparsedStreetAddressContacts as $contactInfo => $contactValue) {
2112         $contactUrl = CRM_Utils_System::url('civicrm/contact/add', 'reset=1&action=update&cid=' . $contactValue['id'], TRUE, NULL, FALSE);
2113         $errorMessage .= "\n Contact ID:" . $contactValue['id'] . " <a href=\"$contactUrl\"> " . $contactValue['streetAddress'] . "</a>";
2114       }
2115       array_unshift($values, $errorMessage);
2116       $returnCode = CRM_Import_Parser::UNPARSED_ADDRESS_WARNING;
2117       $this->setImportStatus((int) ($values[count($values) - 1]), 'ERROR', $errorMessage);
2118     }
2119     return $returnCode;
2120   }
2121
2122   /**
2123    * @param $relKey
2124    * @param array $params
2125    *
2126    * @return bool
2127    */
2128   public function checkRelatedContactFields($relKey, $params) {
2129     //avoid blank contact creation.
2130     $allowToCreate = FALSE;
2131
2132     //build the mapper field array.
2133     static $relatedContactFields = [];
2134     if (!isset($relatedContactFields[$relKey])) {
2135       foreach ($this->_mapperRelated as $key => $name) {
2136         if (!$name) {
2137           continue;
2138         }
2139
2140         if (!empty($relatedContactFields[$name]) && !is_array($relatedContactFields[$name])) {
2141           $relatedContactFields[$name] = [];
2142         }
2143         $fldName = $this->_mapperRelatedContactDetails[$key] ?? NULL;
2144         if ($fldName == 'url') {
2145           $fldName = 'website';
2146         }
2147         if ($fldName) {
2148           $relatedContactFields[$name][] = $fldName;
2149         }
2150       }
2151     }
2152
2153     //validate for passed data.
2154     if (is_array($relatedContactFields[$relKey])) {
2155       foreach ($relatedContactFields[$relKey] as $fld) {
2156         if (!empty($params[$fld])) {
2157           $allowToCreate = TRUE;
2158           break;
2159         }
2160       }
2161     }
2162
2163     return $allowToCreate;
2164   }
2165
2166   /**
2167    * get subtypes given the contact type
2168    *
2169    * @param string $contactType
2170    * @return array $subTypes
2171    */
2172   public static function getSubtypes($contactType) {
2173     $subTypes = [];
2174     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
2175
2176     if (count($types) > 0) {
2177       foreach ($types as $type) {
2178         $subTypes[] = $type['name'];
2179       }
2180     }
2181     return $subTypes;
2182   }
2183
2184   /**
2185    * Get the possible contact matches.
2186    *
2187    * 1) the chosen dedupe rule falling back to
2188    * 2) a check for the external ID.
2189    *
2190    * @see https://issues.civicrm.org/jira/browse/CRM-17275
2191    *
2192    * @param array $params
2193    *
2194    * @return array
2195    *   IDs of possible matches.
2196    *
2197    * @throws \CRM_Core_Exception
2198    * @throws \CiviCRM_API3_Exception
2199    */
2200   protected function getPossibleContactMatches($params) {
2201     $extIDMatch = NULL;
2202
2203     if (!empty($params['external_identifier'])) {
2204       // Check for any match on external id, deleted or otherwise.
2205       $extIDContact = civicrm_api3('Contact', 'get', [
2206         'external_identifier' => $params['external_identifier'],
2207         'showAll' => 'all',
2208         'return' => ['id', 'contact_is_deleted'],
2209       ]);
2210       if (isset($extIDContact['id'])) {
2211         $extIDMatch = $extIDContact['id'];
2212
2213         if ($extIDContact['values'][$extIDMatch]['contact_is_deleted'] == 1) {
2214           // If the contact is deleted, update external identifier to be blank
2215           // to avoid key error from MySQL.
2216           $params = ['id' => $extIDMatch, 'external_identifier' => ''];
2217           civicrm_api3('Contact', 'create', $params);
2218
2219           // And now it is no longer a match.
2220           $extIDMatch = NULL;
2221         }
2222       }
2223     }
2224     $checkParams = ['check_permissions' => FALSE, 'match' => $params];
2225     $checkParams['match']['contact_type'] = $this->_contactType;
2226
2227     $possibleMatches = civicrm_api3('Contact', 'duplicatecheck', $checkParams);
2228     if (!$extIDMatch) {
2229       return array_keys($possibleMatches['values']);
2230     }
2231     if ($possibleMatches['count']) {
2232       if (array_key_exists($extIDMatch, $possibleMatches['values'])) {
2233         return [$extIDMatch];
2234       }
2235       throw new CRM_Core_Exception(ts(
2236         'Matching this contact based on the de-dupe rule would cause an external ID conflict'));
2237     }
2238     return [$extIDMatch];
2239   }
2240
2241   /**
2242    * Format the form mapping parameters ready for the parser.
2243    *
2244    * @param int $count
2245    *   Number of rows.
2246    *
2247    * @return array $parserParameters
2248    */
2249   public static function getParameterForParser($count) {
2250     $baseArray = [];
2251     for ($i = 0; $i < $count; $i++) {
2252       $baseArray[$i] = NULL;
2253     }
2254     $parserParameters['mapperLocType'] = $baseArray;
2255     $parserParameters['mapperPhoneType'] = $baseArray;
2256     $parserParameters['mapperImProvider'] = $baseArray;
2257     $parserParameters['mapperWebsiteType'] = $baseArray;
2258     $parserParameters['mapperRelated'] = $baseArray;
2259     $parserParameters['relatedContactType'] = $baseArray;
2260     $parserParameters['relatedContactDetails'] = $baseArray;
2261     $parserParameters['relatedContactLocType'] = $baseArray;
2262     $parserParameters['relatedContactPhoneType'] = $baseArray;
2263     $parserParameters['relatedContactImProvider'] = $baseArray;
2264     $parserParameters['relatedContactWebsiteType'] = $baseArray;
2265
2266     return $parserParameters;
2267
2268   }
2269
2270   /**
2271    * Set field metadata.
2272    */
2273   protected function setFieldMetadata() {
2274     $this->setImportableFieldsMetadata($this->getContactImportMetadata());
2275     // Probably no longer needed but here for now.
2276     $this->_relationships = $this->getRelationships();
2277   }
2278
2279   /**
2280    * @param array $newContact
2281    * @param $statusFieldName
2282    * @param array $values
2283    * @param int $onDuplicate
2284    * @param array $formatted
2285    * @param array $contactFields
2286    *
2287    * @return int
2288    *
2289    * @throws \CRM_Core_Exception
2290    * @throws \CiviCRM_API3_Exception
2291    * @throws \Civi\API\Exception\UnauthorizedException
2292    */
2293   protected function handleDuplicateError(array $newContact, $statusFieldName, array $values, int $onDuplicate, array $formatted, array $contactFields): int {
2294     $urls = [];
2295     // need to fix at some stage and decide if the error will return an
2296     // array or string, crude hack for now
2297     if (is_array($newContact['error_message']['params'][0])) {
2298       $cids = $newContact['error_message']['params'][0];
2299     }
2300     else {
2301       $cids = explode(',', $newContact['error_message']['params'][0]);
2302     }
2303
2304     foreach ($cids as $cid) {
2305       $urls[] = CRM_Utils_System::url('civicrm/contact/view', 'reset=1&cid=' . $cid, TRUE);
2306     }
2307
2308     $url_string = implode("\n", $urls);
2309
2310     // If we duplicate more than one record, skip no matter what
2311     if (count($cids) > 1) {
2312       $errorMessage = ts('Record duplicates multiple contacts');
2313       //combine error msg to avoid mismatch between error file columns.
2314       $errorMessage .= "\n" . $url_string;
2315       array_unshift($values, $errorMessage);
2316       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2317       return CRM_Import_Parser::ERROR;
2318     }
2319
2320     // Params only had one id, so shift it out
2321     $contactId = array_shift($cids);
2322     $cid = NULL;
2323
2324     $vals = ['contact_id' => $contactId];
2325
2326     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_REPLACE) {
2327       civicrm_api('contact', 'delete', $vals);
2328       $cid = CRM_Contact_BAO_Contact::createProfileContact($formatted, $contactFields, $contactId, NULL, NULL, $formatted['contact_type']);
2329     }
2330     if (in_array((int) $onDuplicate, [CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::DUPLICATE_FILL], TRUE)) {
2331       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactId);
2332     }
2333     // else skip does nothing and just returns an error code.
2334     if ($cid) {
2335       $contact = [
2336         'contact_id' => $cid,
2337       ];
2338       $defaults = [];
2339       $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
2340     }
2341
2342     if (civicrm_error($newContact)) {
2343       if (empty($newContact['error_message']['params'])) {
2344         // different kind of error other than DUPLICATE
2345         $errorMessage = $newContact['error_message'];
2346         array_unshift($values, $errorMessage);
2347         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2348         return CRM_Import_Parser::ERROR;
2349       }
2350
2351       $contactID = $newContact['error_message']['params'][0];
2352       if (is_array($contactID)) {
2353         $contactID = array_pop($contactID);
2354       }
2355       if (!in_array($contactID, $this->_newContacts)) {
2356         $this->_newContacts[] = $contactID;
2357       }
2358     }
2359     //CRM-262 No Duplicate Checking
2360     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
2361       array_unshift($values, $url_string);
2362       $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', 'Skipping duplicate record');
2363       return CRM_Import_Parser::DUPLICATE;
2364     }
2365
2366     $this->setImportStatus((int) $values[count($values) - 1], 'Imported', '');
2367     //return warning if street address is not parsed, CRM-5886
2368     return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID);
2369   }
2370
2371   /**
2372    * Validate a formatted contact parameter list.
2373    *
2374    * @param array $params
2375    *   Structured parameter list (as in crm_format_params).
2376    *
2377    * @throw CRM_Core_Error
2378    */
2379   public function deprecated_validate_formatted_contact(&$params): void {
2380     // Look for offending email addresses
2381
2382     if (array_key_exists('email', $params)) {
2383       foreach ($params['email'] as $count => $values) {
2384         if (!is_array($values)) {
2385           continue;
2386         }
2387         if ($email = CRM_Utils_Array::value('email', $values)) {
2388           // validate each email
2389           if (!CRM_Utils_Rule::email($email)) {
2390             throw new CRM_Core_Exception('No valid email address');
2391           }
2392
2393           // check for loc type id.
2394           if (empty($values['location_type_id'])) {
2395             throw new CRM_Core_Exception('Location Type Id missing.');
2396           }
2397         }
2398       }
2399     }
2400
2401     // Validate custom data fields
2402     if (array_key_exists('custom', $params) && is_array($params['custom'])) {
2403       foreach ($params['custom'] as $key => $custom) {
2404         if (is_array($custom)) {
2405           foreach ($custom as $fieldId => $value) {
2406             $valid = CRM_Core_BAO_CustomValue::typecheck(CRM_Utils_Array::value('type', $value),
2407               CRM_Utils_Array::value('value', $value)
2408             );
2409             if (!$valid && $value['is_required']) {
2410               throw new CRM_Core_Exception('Invalid value for custom field \'' .
2411                 $custom['name'] . '\''
2412               );
2413             }
2414             if (CRM_Utils_Array::value('type', $custom) == 'Date') {
2415               $params['custom'][$key][$fieldId]['value'] = str_replace('-', '', $params['custom'][$key][$fieldId]['value']);
2416             }
2417           }
2418         }
2419       }
2420     }
2421   }
2422
2423   /**
2424    * @param array $params
2425    * @param bool $dupeCheck
2426    * @param null|int $dedupeRuleGroupID
2427    *
2428    * @throws \CRM_Core_Exception
2429    */
2430   public function deprecated_contact_check_params(
2431     &$params,
2432     $dupeCheck = TRUE,
2433     $dedupeRuleGroupID = NULL) {
2434
2435     $requiredCheck = TRUE;
2436
2437     if (isset($params['id']) && is_numeric($params['id'])) {
2438       $requiredCheck = FALSE;
2439     }
2440     if ($requiredCheck) {
2441       $required = [
2442         'Individual' => [
2443           ['first_name', 'last_name'],
2444           'email',
2445         ],
2446         'Household' => [
2447           'household_name',
2448         ],
2449         'Organization' => [
2450           'organization_name',
2451         ],
2452       ];
2453
2454       // contact_type has a limited number of valid values
2455       if (empty($params['contact_type'])) {
2456         throw new CRM_Core_Exception("No Contact Type");
2457       }
2458       $fields = $required[$params['contact_type']] ?? NULL;
2459       if ($fields == NULL) {
2460         throw new CRM_Core_Exception("Invalid Contact Type: {$params['contact_type']}");
2461       }
2462
2463       if ($csType = CRM_Utils_Array::value('contact_sub_type', $params)) {
2464         if (!(CRM_Contact_BAO_ContactType::isExtendsContactType($csType, $params['contact_type']))) {
2465           throw new CRM_Core_Exception("Invalid or Mismatched Contact Subtype: " . implode(', ', (array) $csType));
2466         }
2467       }
2468
2469       if (empty($params['contact_id']) && !empty($params['id'])) {
2470         $valid = FALSE;
2471         $error = '';
2472         foreach ($fields as $field) {
2473           if (is_array($field)) {
2474             $valid = TRUE;
2475             foreach ($field as $element) {
2476               if (empty($params[$element])) {
2477                 $valid = FALSE;
2478                 $error .= $element;
2479                 break;
2480               }
2481             }
2482           }
2483           else {
2484             if (!empty($params[$field])) {
2485               $valid = TRUE;
2486             }
2487           }
2488           if ($valid) {
2489             break;
2490           }
2491         }
2492
2493         if (!$valid) {
2494           throw new CRM_Core_Exception("Required fields not found for {$params['contact_type']} : $error");
2495         }
2496       }
2497     }
2498
2499     if ($dupeCheck) {
2500       // @todo switch to using api version
2501       // $dupes = civicrm_api3('Contact', 'duplicatecheck', (array('match' => $params, 'dedupe_rule_id' => $dedupeRuleGroupID)));
2502       // $ids = $dupes['count'] ? implode(',', array_keys($dupes['values'])) : NULL;
2503       $ids = CRM_Contact_BAO_Contact::getDuplicateContacts($params, $params['contact_type'], 'Unsupervised', [], CRM_Utils_Array::value('check_permissions', $params), $dedupeRuleGroupID);
2504       if ($ids != NULL) {
2505         $error = CRM_Core_Error::createError("Found matching contacts: " . implode(',', $ids),
2506           CRM_Core_Error::DUPLICATE_CONTACT,
2507           'Fatal', $ids
2508         );
2509         return civicrm_api3_create_error($error->pop());
2510       }
2511     }
2512
2513     // check for organisations with same name
2514     if (!empty($params['current_employer'])) {
2515       $organizationParams = ['organization_name' => $params['current_employer']];
2516       $dupeIds = CRM_Contact_BAO_Contact::getDuplicateContacts($organizationParams, 'Organization', 'Supervised', [], FALSE);
2517
2518       // check for mismatch employer name and id
2519       if (!empty($params['employer_id']) && !in_array($params['employer_id'], $dupeIds)
2520       ) {
2521         throw new CRM_Core_Exception('Employer name and Employer id Mismatch');
2522       }
2523
2524       // show error if multiple organisation with same name exist
2525       if (empty($params['employer_id']) && (count($dupeIds) > 1)
2526       ) {
2527         return civicrm_api3_create_error('Found more than one Organisation with same Name.');
2528       }
2529     }
2530   }
2531
2532   /**
2533    * Run import.
2534    *
2535    * @param string $tableName
2536    * @param array $mapper
2537    * @param int $mode
2538    * @param int $contactType
2539    * @param string $primaryKeyName
2540    * @param string $statusFieldName
2541    * @param int $onDuplicate
2542    * @param int $statusID
2543    * @param int $totalRowCount
2544    * @param bool $doGeocodeAddress
2545    * @param int $timeout
2546    * @param string $contactSubType
2547    * @param int $dedupeRuleGroupID
2548    *
2549    * @return mixed
2550    */
2551   public function run(
2552     $tableName,
2553     $mapper = [],
2554     $mode = self::MODE_PREVIEW,
2555     $contactType = self::CONTACT_INDIVIDUAL,
2556     $primaryKeyName = '_id',
2557     $statusFieldName = '_status',
2558     $onDuplicate = self::DUPLICATE_SKIP,
2559     $statusID = NULL,
2560     $totalRowCount = NULL,
2561     $doGeocodeAddress = FALSE,
2562     $timeout = CRM_Contact_Import_Parser_Contact::DEFAULT_TIMEOUT,
2563     $contactSubType = NULL,
2564     $dedupeRuleGroupID = NULL
2565   ) {
2566
2567     // TODO: Make the timeout actually work
2568     $this->_onDuplicate = $onDuplicate;
2569     $this->_dedupeRuleGroupID = $dedupeRuleGroupID;
2570     // Since $this->_contactType is still being called directly do a get call
2571     // here to make sure it is instantiated.
2572     $this->getContactType();
2573     $this->getContactSubType();
2574
2575     $this->init();
2576
2577     $this->_rowCount = 0;
2578     $this->_invalidRowCount = $this->_validCount = 0;
2579     $this->_totalCount = $this->_conflictCount = 0;
2580
2581     $this->_errors = [];
2582     $this->_warnings = [];
2583     $this->_conflicts = [];
2584     $this->_unparsedAddresses = [];
2585
2586     $this->_tableName = $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
2587     $this->_primaryKeyName = '_id';
2588     $this->_statusFieldName = '_status';
2589
2590     if ($mode == self::MODE_MAPFIELD) {
2591       $this->_rows = [];
2592     }
2593     else {
2594       $this->_activeFieldCount = count($this->_activeFields);
2595     }
2596
2597     if ($statusID) {
2598       $this->progressImport($statusID);
2599       $startTimestamp = $currTimestamp = $prevTimestamp = time();
2600     }
2601     // get the contents of the temp. import table
2602     $query = "SELECT * FROM $tableName";
2603     if ($mode == self::MODE_IMPORT) {
2604       $query .= " WHERE _status = 'NEW'";
2605     }
2606     if ($this->_maxLinesToProcess > 0) {
2607       // Note this would only be the case in MapForm mode, where it is set to 100
2608       // rows. In fact mapField really only needs 2 rows - the reason for
2609       // 100 seems to be that the other import classes are processing a
2610       // csv file, and there was a concern that some rows might have more
2611       // columns than others - hence checking 100 rows perhaps seemed like
2612       // a good precaution presumably when determining the activeFieldsCount
2613       // which is the number of columns a row might have.
2614       // However, the mapField class may no longer use activeFieldsCount for contact
2615       // to be continued....
2616       $query .= ' LIMIT ' . $this->_maxLinesToProcess;
2617     }
2618
2619     $result = CRM_Core_DAO::executeQuery($query);
2620
2621     while ($result->fetch()) {
2622       $values = array_values($result->toArray());
2623       $this->_rowCount++;
2624
2625       /* trim whitespace around the values */
2626       foreach ($values as $k => $v) {
2627         $values[$k] = trim($v, " \t\r\n");
2628       }
2629
2630       $this->_totalCount++;
2631
2632       if ($mode == self::MODE_MAPFIELD) {
2633         $returnCode = CRM_Import_Parser::VALID;
2634       }
2635       elseif ($mode == self::MODE_PREVIEW) {
2636         $returnCode = $this->preview($values);
2637       }
2638       elseif ($mode == self::MODE_SUMMARY) {
2639         $returnCode = $this->summary($values);
2640       }
2641       elseif ($mode == self::MODE_IMPORT) {
2642         try {
2643           $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
2644         }
2645         catch (CiviCRM_API3_Exception $e) {
2646           // When we catch errors here we are not adding to the errors array - mostly
2647           // because that will become obsolete once https://github.com/civicrm/civicrm-core/pull/23292
2648           // is merged and this will replace it as the main way to handle errors (ie. update the table
2649           // and move on).
2650           $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $e->getMessage());
2651         }
2652         if ($statusID && (($this->_rowCount % 50) == 0)) {
2653           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
2654         }
2655       }
2656       else {
2657         $returnCode = self::ERROR;
2658       }
2659
2660       // note that a line could be valid but still produce a warning
2661       if ($returnCode & self::VALID) {
2662         $this->_validCount++;
2663         if ($mode == self::MODE_MAPFIELD) {
2664           $this->_rows[] = $values;
2665           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
2666         }
2667       }
2668
2669       if ($returnCode & self::ERROR) {
2670         $this->_invalidRowCount++;
2671         array_unshift($values, $this->_rowCount);
2672         $this->_errors[] = $values;
2673       }
2674
2675       if ($returnCode & self::CONFLICT) {
2676         $this->_conflictCount++;
2677         array_unshift($values, $this->_rowCount);
2678         $this->_conflicts[] = $values;
2679       }
2680
2681       if ($returnCode & self::NO_MATCH) {
2682         $this->_unMatchCount++;
2683         array_unshift($values, $this->_rowCount);
2684         $this->_unMatch[] = $values;
2685       }
2686
2687       if ($returnCode & self::DUPLICATE) {
2688         $this->_duplicateCount++;
2689         array_unshift($values, $this->_rowCount);
2690         $this->_duplicates[] = $values;
2691         if ($onDuplicate != self::DUPLICATE_SKIP) {
2692           $this->_validCount++;
2693         }
2694       }
2695
2696       if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
2697         $this->_unparsedAddressCount++;
2698         array_unshift($values, $this->_rowCount);
2699         $this->_unparsedAddresses[] = $values;
2700       }
2701
2702       // see if we've hit our timeout yet
2703       /* if ( $the_thing_with_the_stuff ) {
2704       do_something( );
2705       } */
2706     }
2707
2708     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
2709       $customHeaders = $mapper;
2710
2711       $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType);
2712       foreach ($customHeaders as $key => $value) {
2713         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
2714           $customHeaders[$key] = $customfields[$id][0];
2715         }
2716       }
2717
2718       if ($this->_conflictCount) {
2719         $headers = array_merge([
2720           ts('Line Number'),
2721           ts('Reason'),
2722         ], $customHeaders);
2723         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
2724         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
2725       }
2726       if ($this->_unMatchCount) {
2727         $headers = array_merge([
2728           ts('Line Number'),
2729           ts('Reason'),
2730         ], $customHeaders);
2731
2732         $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH);
2733         self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch);
2734       }
2735       if ($this->_unparsedAddressCount) {
2736         $headers = array_merge([
2737           ts('Line Number'),
2738           ts('Contact Edit URL'),
2739         ], $customHeaders);
2740         $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING);
2741         self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses);
2742       }
2743     }
2744   }
2745
2746   /**
2747    * Given a list of the importable field keys that the user has selected.
2748    * set the active fields array to this list
2749    *
2750    * @param array $fieldKeys
2751    *   Mapped array of values.
2752    */
2753   public function setActiveFields($fieldKeys) {
2754     $this->_activeFieldCount = count($fieldKeys);
2755     foreach ($fieldKeys as $key) {
2756       if (empty($this->_fields[$key])) {
2757         $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
2758       }
2759       else {
2760         $this->_activeFields[] = clone($this->_fields[$key]);
2761       }
2762     }
2763   }
2764
2765   /**
2766    * @param $elements
2767    */
2768   public function setActiveFieldLocationTypes($elements) {
2769     for ($i = 0; $i < count($elements); $i++) {
2770       $this->_activeFields[$i]->_hasLocationType = $elements[$i];
2771     }
2772   }
2773
2774   /**
2775    * @param $elements
2776    */
2777
2778   /**
2779    * @param $elements
2780    */
2781   public function setActiveFieldPhoneTypes($elements) {
2782     for ($i = 0; $i < count($elements); $i++) {
2783       $this->_activeFields[$i]->_phoneType = $elements[$i];
2784     }
2785   }
2786
2787   /**
2788    * @param $elements
2789    */
2790   public function setActiveFieldWebsiteTypes($elements) {
2791     for ($i = 0; $i < count($elements); $i++) {
2792       $this->_activeFields[$i]->_websiteType = $elements[$i];
2793     }
2794   }
2795
2796   /**
2797    * Set IM Service Provider type fields.
2798    *
2799    * @param array $elements
2800    *   IM service provider type ids.
2801    */
2802   public function setActiveFieldImProviders($elements) {
2803     for ($i = 0; $i < count($elements); $i++) {
2804       $this->_activeFields[$i]->_imProvider = $elements[$i];
2805     }
2806   }
2807
2808   /**
2809    * @param $elements
2810    */
2811   public function setActiveFieldRelated($elements) {
2812     for ($i = 0; $i < count($elements); $i++) {
2813       $this->_activeFields[$i]->_related = $elements[$i];
2814     }
2815   }
2816
2817   /**
2818    * @param $elements
2819    */
2820   public function setActiveFieldRelatedContactType($elements) {
2821     for ($i = 0; $i < count($elements); $i++) {
2822       $this->_activeFields[$i]->_relatedContactType = $elements[$i];
2823     }
2824   }
2825
2826   /**
2827    * @param $elements
2828    */
2829   public function setActiveFieldRelatedContactDetails($elements) {
2830     for ($i = 0; $i < count($elements); $i++) {
2831       $this->_activeFields[$i]->_relatedContactDetails = $elements[$i];
2832     }
2833   }
2834
2835   /**
2836    * @param $elements
2837    */
2838   public function setActiveFieldRelatedContactLocType($elements) {
2839     for ($i = 0; $i < count($elements); $i++) {
2840       $this->_activeFields[$i]->_relatedContactLocType = $elements[$i];
2841     }
2842   }
2843
2844   /**
2845    * Set active field for related contact's phone type.
2846    *
2847    * @param array $elements
2848    */
2849   public function setActiveFieldRelatedContactPhoneType($elements) {
2850     for ($i = 0; $i < count($elements); $i++) {
2851       $this->_activeFields[$i]->_relatedContactPhoneType = $elements[$i];
2852     }
2853   }
2854
2855   /**
2856    * @param $elements
2857    */
2858   public function setActiveFieldRelatedContactWebsiteType($elements) {
2859     for ($i = 0; $i < count($elements); $i++) {
2860       $this->_activeFields[$i]->_relatedContactWebsiteType = $elements[$i];
2861     }
2862   }
2863
2864   /**
2865    * Set IM Service Provider type fields for related contacts.
2866    *
2867    * @param array $elements
2868    *   IM service provider type ids of related contact.
2869    */
2870   public function setActiveFieldRelatedContactImProvider($elements) {
2871     for ($i = 0; $i < count($elements); $i++) {
2872       $this->_activeFields[$i]->_relatedContactImProvider = $elements[$i];
2873     }
2874   }
2875
2876   /**
2877    * Format the field values for input to the api.
2878    *
2879    * @return array
2880    *   (reference ) associative array of name/value pairs
2881    */
2882   public function &getActiveFieldParams() {
2883     $params = [];
2884
2885     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
2886       $fieldName = $this->_activeFields[$i]->_name;
2887       if ($fieldName === 'do_not_import') {
2888         continue;
2889       }
2890       $relatedContactFieldName = $this->_activeFields[$i]->_relatedContactDetails;
2891       $relatedContactType = $this->_activeFields[$i]->_relatedContactType;
2892       $relatedContactLocationTypeID = $this->_activeFields[$i]->_relatedContactLocType;
2893       $relatedContactWebsiteTypeID = $this->_activeFields[$i]->_relatedContactWebsiteType ?? NULL;
2894       $relatedContactIMProviderID = $this->_activeFields[$i]->_relatedContactImProvider ?? NULL;
2895       $relatedContactPhoneTypeID = $this->_activeFields[$i]->_relatedContactPhoneType ?? NULL;
2896       // The key is in the format 5_a_b where 5 is the relationship_type_id and a_b is the direction.
2897       $relatedContactKey = $this->_activeFields[$i]->_related;
2898
2899       $locationTypeID = $this->_activeFields[$i]->_hasLocationType;
2900       $phoneTypeID = $this->_activeFields[$i]->_phoneType;
2901       $imProviderID = $this->_activeFields[$i]->_imProvider ?? NULL;
2902       $websiteTypeID = $this->_activeFields[$i]->_websiteType ?? NULL;
2903
2904       $importedValue = $this->_activeFields[$i]->_value;
2905
2906       if (isset($importedValue)) {
2907         if (isset($locationTypeID)) {
2908           if (!isset($params[$fieldName])) {
2909             $params[$fieldName] = [];
2910           }
2911
2912           $value = [
2913             $fieldName => $importedValue,
2914             'location_type_id' => $locationTypeID,
2915           ];
2916
2917           if (isset($phoneTypeID)) {
2918             $value['phone_type_id'] = $phoneTypeID;
2919           }
2920
2921           // get IM service Provider type id
2922           if (isset($imProviderID)) {
2923             $value['provider_id'] = $imProviderID;
2924           }
2925
2926           $params[$fieldName][] = $value;
2927         }
2928         elseif (isset($websiteTypeID)) {
2929           $value = [
2930             $fieldName => $importedValue,
2931             'website_type_id' => $websiteTypeID,
2932           ];
2933
2934           $params[$fieldName][] = $value;
2935         }
2936
2937         if (!isset($params[$fieldName])) {
2938           if (!isset($relatedContactKey)) {
2939             $params[$fieldName] = $importedValue;
2940           }
2941         }
2942
2943         //minor fix for CRM-4062
2944         if (isset($relatedContactKey)) {
2945           if (!isset($params[$relatedContactKey])) {
2946             $params[$relatedContactKey] = [];
2947           }
2948
2949           if (!isset($params[$relatedContactKey]['contact_type']) && !empty($relatedContactType)) {
2950             $params[$relatedContactKey]['contact_type'] = $relatedContactType;
2951           }
2952
2953           if (isset($relatedContactLocationTypeID) && !empty($importedValue)) {
2954             if (!empty($params[$relatedContactKey][$relatedContactFieldName]) &&
2955               !is_array($params[$relatedContactKey][$relatedContactFieldName])
2956             ) {
2957               $params[$relatedContactKey][$relatedContactFieldName] = [];
2958             }
2959             $value = [
2960               $relatedContactFieldName => $importedValue,
2961               'location_type_id' => $relatedContactLocationTypeID,
2962             ];
2963
2964             if (isset($relatedContactPhoneTypeID)) {
2965               $value['phone_type_id'] = $relatedContactPhoneTypeID;
2966             }
2967
2968             // get IM service Provider type id for related contact
2969             if (isset($relatedContactIMProviderID)) {
2970               $value['provider_id'] = $relatedContactIMProviderID;
2971             }
2972
2973             $params[$relatedContactKey][$relatedContactFieldName][] = $value;
2974           }
2975           elseif (isset($relatedContactWebsiteTypeID)) {
2976             $params[$relatedContactKey][$relatedContactFieldName][] = [
2977               'url' => $importedValue,
2978               'website_type_id' => $relatedContactWebsiteTypeID,
2979             ];
2980           }
2981           elseif (empty($importedValue) && isset($relatedContactLocationTypeID)) {
2982             if (empty($params[$relatedContactKey][$relatedContactFieldName])) {
2983               $params[$relatedContactKey][$relatedContactFieldName] = [];
2984             }
2985           }
2986           else {
2987             $params[$relatedContactKey][$relatedContactFieldName] = $importedValue;
2988           }
2989         }
2990       }
2991     }
2992
2993     return $params;
2994   }
2995
2996   /**
2997    * @param string $name
2998    * @param $title
2999    * @param int $type
3000    * @param string $headerPattern
3001    * @param string $dataPattern
3002    * @param bool $hasLocationType
3003    */
3004   public function addField(
3005     $name, $title, $type = CRM_Utils_Type::T_INT,
3006     $headerPattern = '//', $dataPattern = '//',
3007     $hasLocationType = FALSE
3008   ) {
3009     $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
3010     if (empty($name)) {
3011       $this->_fields['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
3012     }
3013   }
3014
3015   /**
3016    * Store parser values.
3017    *
3018    * @param CRM_Core_Session $store
3019    *
3020    * @param int $mode
3021    */
3022   public function set($store, $mode = self::MODE_SUMMARY) {
3023     // @todo - this params are being set here because they were / possibly still
3024     // are in some places being accessed by forms later in the flow
3025     // ie CRM_Contact_Import_Form_MapField, CRM_Contact_Import_Form_Preview
3026     // or CRM_Contact_Import_Form_Summary using `$this->get()
3027     // which was the old way of saving values submitted on this form such that
3028     // the other forms could access them. Now they should use
3029     // `getSubmittedValue` or simply not get them if the only
3030     // reason is to pass to the Parser which can itself
3031     // call 'getSubmittedValue'
3032     // Once the mentioned forms no longer call $this->get() all this 'setting'
3033     // is obsolete.
3034     $store->set('rowCount', $this->_rowCount);
3035     $store->set('fieldTypes', $this->getSelectTypes());
3036
3037     $store->set('columnCount', $this->_activeFieldCount);
3038
3039     $store->set('totalRowCount', $this->_totalCount);
3040     $store->set('validRowCount', $this->_validCount);
3041     $store->set('invalidRowCount', $this->_invalidRowCount);
3042     $store->set('conflictRowCount', $this->_conflictCount);
3043     $store->set('unMatchCount', $this->_unMatchCount);
3044
3045     switch ($this->_contactType) {
3046       case 'Individual':
3047         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
3048         break;
3049
3050       case 'Household':
3051         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
3052         break;
3053
3054       case 'Organization':
3055         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
3056     }
3057
3058     if ($this->_conflictCount) {
3059       $store->set('conflictsFileName', $this->_conflictFileName);
3060     }
3061     if (isset($this->_rows) && !empty($this->_rows)) {
3062       $store->set('dataValues', $this->_rows);
3063     }
3064
3065     if ($this->_unMatchCount) {
3066       $store->set('mismatchFileName', $this->_misMatchFilemName);
3067     }
3068
3069     if ($mode == self::MODE_IMPORT) {
3070       $store->set('duplicateRowCount', $this->_duplicateCount);
3071       $store->set('unparsedAddressCount', $this->_unparsedAddressCount);
3072       if ($this->_duplicateCount) {
3073         $store->set('duplicatesFileName', $this->_duplicateFileName);
3074       }
3075     }
3076   }
3077
3078   /**
3079    * Export data to a CSV file.
3080    *
3081    * @param string $fileName
3082    * @param array $header
3083    * @param array $data
3084    */
3085   public static function exportCSV($fileName, $header, $data) {
3086
3087     if (file_exists($fileName) && !is_writable($fileName)) {
3088       CRM_Core_Error::movedSiteError($fileName);
3089     }
3090     //hack to remove '_status', '_statusMsg' and '_id' from error file
3091     $errorValues = [];
3092     $dbRecordStatus = ['IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW'];
3093     foreach ($data as $rowCount => $rowValues) {
3094       $count = 0;
3095       foreach ($rowValues as $key => $val) {
3096         if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
3097           break;
3098         }
3099         $errorValues[$rowCount][$key] = $val;
3100         $count++;
3101       }
3102     }
3103     $data = $errorValues;
3104
3105     $output = [];
3106     $fd = fopen($fileName, 'w');
3107
3108     foreach ($header as $key => $value) {
3109       $header[$key] = "\"$value\"";
3110     }
3111     $config = CRM_Core_Config::singleton();
3112     $output[] = implode($config->fieldSeparator, $header);
3113
3114     foreach ($data as $datum) {
3115       foreach ($datum as $key => $value) {
3116         $datum[$key] = "\"$value\"";
3117       }
3118       $output[] = implode($config->fieldSeparator, $datum);
3119     }
3120     fwrite($fd, implode("\n", $output));
3121     fclose($fd);
3122   }
3123
3124   /**
3125    * Set the import status for the given record.
3126    *
3127    * If this is a sql import then the sql table will be used and the update
3128    * will not happen as the relevant fields don't exist in the table - hence
3129    * the checks that statusField & primary key are set.
3130    *
3131    * @param int $id
3132    * @param string $status
3133    * @param string $message
3134    */
3135   public function setImportStatus(int $id, string $status, string $message): void {
3136     if ($this->_statusFieldName && $this->_primaryKeyName) {
3137       CRM_Core_DAO::executeQuery("
3138         UPDATE $this->_tableName
3139         SET $this->_statusFieldName = %1,
3140           {$this->_statusFieldName}Msg = %2
3141         WHERE  $this->_primaryKeyName = %3
3142       ", [
3143         1 => [$status, 'String'],
3144         2 => [$message, 'String'],
3145         3 => [$id, 'Integer'],
3146       ]);
3147     }
3148   }
3149
3150   /**
3151    * Format contact parameters.
3152    *
3153    * @todo this function needs re-writing & re-merging into the main function.
3154    *
3155    * Here be dragons.
3156    *
3157    * @param array $values
3158    * @param array $params
3159    *
3160    * @return bool
3161    */
3162   protected function formatContactParameters(&$values, &$params) {
3163     // Crawl through the possible classes:
3164     // Contact
3165     //      Individual
3166     //      Household
3167     //      Organization
3168     //          Location
3169     //              Address
3170     //              Email
3171     //              Phone
3172     //              IM
3173     //      Note
3174     //      Custom
3175
3176     // first add core contact values since for other Civi modules they are not added
3177     $contactFields = CRM_Contact_DAO_Contact::fields();
3178     _civicrm_api3_store_values($contactFields, $values, $params);
3179
3180     if (isset($values['contact_type'])) {
3181       // we're an individual/household/org property
3182
3183       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
3184
3185       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
3186       return TRUE;
3187     }
3188
3189     // Cache the various object fields
3190     // @todo - remove this after confirming this is just a compilation of other-wise-cached fields.
3191     static $fields = [];
3192
3193     if (isset($values['individual_prefix'])) {
3194       if (!empty($params['prefix_id'])) {
3195         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
3196         $params['prefix'] = $prefixes[$params['prefix_id']];
3197       }
3198       else {
3199         $params['prefix'] = $values['individual_prefix'];
3200       }
3201       return TRUE;
3202     }
3203
3204     if (isset($values['individual_suffix'])) {
3205       if (!empty($params['suffix_id'])) {
3206         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
3207         $params['suffix'] = $suffixes[$params['suffix_id']];
3208       }
3209       else {
3210         $params['suffix'] = $values['individual_suffix'];
3211       }
3212       return TRUE;
3213     }
3214
3215     // CRM-4575
3216     if (isset($values['email_greeting'])) {
3217       if (!empty($params['email_greeting_id'])) {
3218         $emailGreetingFilter = [
3219           'contact_type' => $params['contact_type'] ?? NULL,
3220           'greeting_type' => 'email_greeting',
3221         ];
3222         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
3223         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
3224       }
3225       else {
3226         $params['email_greeting'] = $values['email_greeting'];
3227       }
3228
3229       return TRUE;
3230     }
3231
3232     if (isset($values['postal_greeting'])) {
3233       if (!empty($params['postal_greeting_id'])) {
3234         $postalGreetingFilter = [
3235           'contact_type' => $params['contact_type'] ?? NULL,
3236           'greeting_type' => 'postal_greeting',
3237         ];
3238         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
3239         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
3240       }
3241       else {
3242         $params['postal_greeting'] = $values['postal_greeting'];
3243       }
3244       return TRUE;
3245     }
3246
3247     if (isset($values['addressee'])) {
3248       $params['addressee'] = $values['addressee'];
3249       return TRUE;
3250     }
3251
3252     if (isset($values['gender'])) {
3253       if (!empty($params['gender_id'])) {
3254         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
3255         $params['gender'] = $genders[$params['gender_id']];
3256       }
3257       else {
3258         $params['gender'] = $values['gender'];
3259       }
3260       return TRUE;
3261     }
3262
3263     if (!empty($values['preferred_communication_method'])) {
3264       $comm = [];
3265       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
3266
3267       $preffComm = explode(',', $values['preferred_communication_method']);
3268       foreach ($preffComm as $v) {
3269         $v = strtolower(trim($v));
3270         if (array_key_exists($v, $pcm)) {
3271           $comm[$pcm[$v]] = 1;
3272         }
3273       }
3274
3275       $params['preferred_communication_method'] = $comm;
3276       return TRUE;
3277     }
3278
3279     // format the website params.
3280     if (!empty($values['url'])) {
3281       static $websiteFields;
3282       if (!is_array($websiteFields)) {
3283         $websiteFields = CRM_Core_DAO_Website::fields();
3284       }
3285       if (!array_key_exists('website', $params) ||
3286         !is_array($params['website'])
3287       ) {
3288         $params['website'] = [];
3289       }
3290
3291       $websiteCount = count($params['website']);
3292       _civicrm_api3_store_values($websiteFields, $values,
3293         $params['website'][++$websiteCount]
3294       );
3295
3296       return TRUE;
3297     }
3298
3299     if (isset($values['note'])) {
3300       // add a note field
3301       if (!isset($params['note'])) {
3302         $params['note'] = [];
3303       }
3304       $noteBlock = count($params['note']) + 1;
3305
3306       $params['note'][$noteBlock] = [];
3307       if (!isset($fields['Note'])) {
3308         $fields['Note'] = CRM_Core_DAO_Note::fields();
3309       }
3310
3311       // get the current logged in civicrm user
3312       $session = CRM_Core_Session::singleton();
3313       $userID = $session->get('userID');
3314
3315       if ($userID) {
3316         $values['contact_id'] = $userID;
3317       }
3318
3319       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
3320
3321       return TRUE;
3322     }
3323
3324     // Check for custom field values
3325     $customFields = CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
3326       FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
3327     );
3328
3329     foreach ($values as $key => $value) {
3330       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
3331         // check if it's a valid custom field id
3332
3333         if (!array_key_exists($customFieldID, $customFields)) {
3334           return civicrm_api3_create_error('Invalid custom field ID');
3335         }
3336         else {
3337           $params[$key] = $value;
3338         }
3339       }
3340     }
3341     return TRUE;
3342   }
3343
3344   /**
3345    * Format location block ready for importing.
3346    *
3347    * There is some test coverage for this in CRM_Contact_Import_Parser_ContactTest
3348    * e.g. testImportPrimaryAddress.
3349    *
3350    * @param array $values
3351    * @param array $params
3352    *
3353    * @return bool
3354    */
3355   protected function formatLocationBlock(&$values, &$params) {
3356     $blockTypes = [
3357       'phone' => 'Phone',
3358       'email' => 'Email',
3359       'im' => 'IM',
3360       'openid' => 'OpenID',
3361       'phone_ext' => 'Phone',
3362     ];
3363     foreach ($blockTypes as $blockFieldName => $block) {
3364       if (!array_key_exists($blockFieldName, $values)) {
3365         continue;
3366       }
3367       $blockIndex = $values['location_type_id'] . (!empty($values['phone_type_id']) ? '_' . $values['phone_type_id'] : '');
3368
3369       // block present in value array.
3370       if (!array_key_exists($blockFieldName, $params) || !is_array($params[$blockFieldName])) {
3371         $params[$blockFieldName] = [];
3372       }
3373
3374       $fields[$block] = $this->getMetadataForEntity($block);
3375
3376       // copy value to dao field name.
3377       if ($blockFieldName == 'im') {
3378         $values['name'] = $values[$blockFieldName];
3379       }
3380
3381       _civicrm_api3_store_values($fields[$block], $values,
3382         $params[$blockFieldName][$blockIndex]
3383       );
3384
3385       $this->fillPrimary($params[$blockFieldName][$blockIndex], $values, $block, CRM_Utils_Array::value('id', $params));
3386
3387       if (empty($params['id']) && (count($params[$blockFieldName]) == 1)) {
3388         $params[$blockFieldName][$blockIndex]['is_primary'] = TRUE;
3389       }
3390
3391       // we only process single block at a time.
3392       return TRUE;
3393     }
3394
3395     // handle address fields.
3396     if (!array_key_exists('address', $params) || !is_array($params['address'])) {
3397       $params['address'] = [];
3398     }
3399
3400     // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
3401     // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
3402     // the address in CRM_Core_BAO_Address::create method
3403     if (!empty($values['location_type_id'])) {
3404       static $customFields = [];
3405       if (empty($customFields)) {
3406         $customFields = CRM_Core_BAO_CustomField::getFields('Address');
3407       }
3408       // make a copy of values, as we going to make changes
3409       $newValues = $values;
3410       foreach ($values as $key => $val) {
3411         $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
3412         if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
3413
3414           $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
3415           if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
3416             $mulValues = explode(',', $val);
3417             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
3418             $newValues[$key] = [];
3419             foreach ($mulValues as $v1) {
3420               foreach ($customOption as $v2) {
3421                 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
3422                   (strtolower($v2['value']) == strtolower(trim($v1)))
3423                 ) {
3424                   if ($htmlType == 'CheckBox') {
3425                     $newValues[$key][$v2['value']] = 1;
3426                   }
3427                   else {
3428                     $newValues[$key][] = $v2['value'];
3429                   }
3430                 }
3431               }
3432             }
3433           }
3434         }
3435       }
3436       // consider new values
3437       $values = $newValues;
3438     }
3439
3440     $fields['Address'] = $this->getMetadataForEntity('Address');
3441     // @todo this is kinda replicated below....
3442     _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$values['location_type_id']]);
3443
3444     $addressFields = [
3445       'county',
3446       'country',
3447       'state_province',
3448       'supplemental_address_1',
3449       'supplemental_address_2',
3450       'supplemental_address_3',
3451       'StateProvince.name',
3452     ];
3453     foreach (array_keys($customFields) as $customFieldID) {
3454       $addressFields[] = 'custom_' . $customFieldID;
3455     }
3456
3457     foreach ($addressFields as $field) {
3458       if (array_key_exists($field, $values)) {
3459         if (!array_key_exists('address', $params)) {
3460           $params['address'] = [];
3461         }
3462         $params['address'][$values['location_type_id']][$field] = $values[$field];
3463       }
3464     }
3465
3466     $this->fillPrimary($params['address'][$values['location_type_id']], $values, 'address', CRM_Utils_Array::value('id', $params));
3467     return TRUE;
3468   }
3469
3470   /**
3471    * Get the field metadata for the relevant entity.
3472    *
3473    * @param string $entity
3474    *
3475    * @return array
3476    */
3477   protected function getMetadataForEntity($entity) {
3478     if (!isset($this->fieldMetadata[$entity])) {
3479       $className = "CRM_Core_DAO_$entity";
3480       $this->fieldMetadata[$entity] = $className::fields();
3481     }
3482     return $this->fieldMetadata[$entity];
3483   }
3484
3485   /**
3486    * Fill in the primary location.
3487    *
3488    * If the contact has a primary address we update it. Otherwise
3489    * we add an address of the default location type.
3490    *
3491    * @param array $params
3492    *   Address block parameters
3493    * @param array $values
3494    *   Input values
3495    * @param string $entity
3496    *  - address, email, phone
3497    * @param int|null $contactID
3498    *
3499    * @throws \CiviCRM_API3_Exception
3500    */
3501   protected function fillPrimary(&$params, $values, $entity, $contactID) {
3502     if ($values['location_type_id'] === 'Primary') {
3503       if ($contactID) {
3504         $primary = civicrm_api3($entity, 'get', [
3505           'return' => 'location_type_id',
3506           'contact_id' => $contactID,
3507           'is_primary' => 1,
3508           'sequential' => 1,
3509         ]);
3510       }
3511       $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
3512       $params['location_type_id'] = (int) (isset($primary) && $primary['count']) ? $primary['values'][0]['location_type_id'] : $defaultLocationType->id;
3513       $params['is_primary'] = 1;
3514     }
3515   }
3516
3517   /**
3518    * Get the civicrm_mapping_field appropriate layout for the mapper input.
3519    *
3520    * The input looks something like ['street_address', 1]
3521    * and would be mapped to ['name' => 'street_address', 'location_type_id' =>
3522    * 1]
3523    *
3524    * @param array $fieldMapping
3525    * @param int $mappingID
3526    * @param int $columnNumber
3527    *
3528    * @return array
3529    * @throws \API_Exception
3530    */
3531   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
3532     $isRelationshipField = preg_match('/\d*_a_b|b_a$/', $fieldMapping[0]);
3533     $fieldName = $isRelationshipField ? $fieldMapping[1] : $fieldMapping[0];
3534     $locationTypeID = NULL;
3535     $possibleLocationField = $isRelationshipField ? 2 : 1;
3536     if ($fieldName !== 'url' && is_numeric($fieldMapping[$possibleLocationField] ?? NULL)) {
3537       $locationTypeID = $fieldMapping[$possibleLocationField];
3538     }
3539     return [
3540       'name' => $fieldName,
3541       'mapping_id' => $mappingID,
3542       'relationship_type_id' => $isRelationshipField ? substr($fieldMapping[0], 0, -4) : NULL,
3543       'relationship_direction' => $isRelationshipField ? substr($fieldMapping[0], -3) : NULL,
3544       'column_number' => $columnNumber,
3545       'contact_type' => $this->getContactType(),
3546       'website_type_id' => $fieldName !== 'url' ? NULL : ($isRelationshipField ? $fieldMapping[2] : $fieldMapping[1]),
3547       'phone_type_id' => $fieldName !== 'phone' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
3548       'im_provider_id' => $fieldName !== 'im' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
3549       'location_type_id' => $locationTypeID,
3550     ];
3551   }
3552
3553   /**
3554    * @param array $mappedField
3555    *   Field detail as would be saved in field_mapping table
3556    *   or as returned from getMappingFieldFromMapperInput
3557    *
3558    * @return string
3559    * @throws \API_Exception
3560    */
3561   public function getMappedFieldLabel(array $mappedField): string {
3562     $this->setFieldMetadata();
3563     $title = [];
3564     if ($mappedField['relationship_type_id']) {
3565       $title[] = $this->getRelationshipLabel($mappedField['relationship_type_id'], $mappedField['relationship_direction']);
3566     }
3567     $title[] = $this->getImportableFieldsMetadata()[$mappedField['name']]['title'];
3568     if ($mappedField['location_type_id']) {
3569       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Address', 'location_type_id', $mappedField['location_type_id']);
3570     }
3571     if ($mappedField['website_type_id']) {
3572       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Website', 'website_type_id', $mappedField['website_type_id']);
3573     }
3574     if ($mappedField['phone_type_id']) {
3575       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $mappedField['phone_type_id']);
3576     }
3577     if ($mappedField['im_provider_id']) {
3578       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $mappedField['provider_id']);
3579     }
3580     return implode(' - ', $title);
3581   }
3582
3583   /**
3584    * Get the relevant label for the relationship.
3585    *
3586    * @param int $id
3587    * @param string $direction
3588    *
3589    * @return string
3590    * @throws \API_Exception
3591    */
3592   protected function getRelationshipLabel(int $id, string $direction): string {
3593     if (empty($this->relationshipLabels[$id . $direction])) {
3594       $this->relationshipLabels[$id . $direction] =
3595       $fieldName = 'label_' . $direction;
3596       $this->relationshipLabels[$id . $direction] = (string) RelationshipType::get(FALSE)
3597         ->addWhere('id', '=', $id)
3598         ->addSelect($fieldName)->execute()->first()[$fieldName];
3599     }
3600     return $this->relationshipLabels[$id . $direction];
3601   }
3602
3603   /**
3604    * Transform the input parameters into the form handled by the input routine.
3605    *
3606    * @param array $values
3607    *   Input parameters as they come in from the datasource
3608    *   eg. ['Bob', 'Smith', 'bob@example.org', '123-456']
3609    *
3610    * @return array
3611    *   Parameters mapped to CiviCRM fields based on the mapping
3612    *   and specified contact type. eg.
3613    *   [
3614    *     'contact_type' => 'Individual',
3615    *     'first_name' => 'Bob',
3616    *     'last_name' => 'Smith',
3617    *     'phone' => ['phone' => '123', 'location_type_id' => 1, 'phone_type_id' => 1],
3618    *     '5_a_b' => ['contact_type' => 'Organization', 'url' => ['url' => 'https://example.org', 'website_type_id' => 1]]
3619    *     'im' => ['im' => 'my-handle', 'location_type_id' => 1, 'provider_id' => 1],
3620    *
3621    * @throws \API_Exception
3622    */
3623   public function getMappedRow(array $values): array {
3624     $this->setActiveFieldValues($values);
3625     $params = $this->getActiveFieldParams();
3626     $params['contact_type'] = $this->getContactType();
3627     return $params;
3628   }
3629
3630   /**
3631    * Is the job complete.
3632    *
3633    * This function transitionally accesses the table from the userJob
3634    * directly - but the function should be moved to the dataSource class.
3635    *
3636    * @throws \API_Exception
3637    */
3638   public function isComplete() {
3639     $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
3640     return (bool) CRM_Core_DAO::singleValueQuery("SELECT count(*) FROM $tableName WHERE _status = 'NEW' LIMIT 1");
3641   }
3642
3643 }