CRM/Contact/Import/Parser/Contact.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 use Civi\Api4\Contact;
  13 use Civi\Api4\RelationshipType;
  14
  15 require_once 'CRM/Utils/DeprecatedUtils.php';
  16 require_once 'api/v3/utils.php';
  17
  18 /**
  19  *
  20  * @package CRM
  21  * @copyright CiviCRM LLC https://civicrm.org/licensing
  22  */
  23
  24 /**
  25  * class to parse contact csv files
  26  */
  27 class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser {
  28
  29   use CRM_Contact_Import_MetadataTrait;
  30
  31   protected $_mapperKeys = [];
  32   protected $_mapperLocType = [];
  33   protected $_mapperPhoneType;
  34   protected $_mapperImProvider;
  35   protected $_mapperWebsiteType;
  36   protected $_mapperRelated;
  37   protected $_mapperRelatedContactType;
  38   protected $_mapperRelatedContactDetails;
  39   protected $_mapperRelatedContactEmailType;
  40   protected $_mapperRelatedContactImProvider;
  41   protected $_mapperRelatedContactWebsiteType;
  42   protected $_relationships;
  43
  44   protected $_emailIndex;
  45   protected $_firstNameIndex;
  46   protected $_lastNameIndex;
  47
  48   protected $_householdNameIndex;
  49   protected $_organizationNameIndex;
  50
  51   protected $_allEmails;
  52
  53   protected $_phoneIndex;
  54
  55   /**
  56    * Is update only permitted on an id match.
  57    *
  58    * Note this historically was true for when id or external identifier was
  59    * present. However, CRM-17275 determined that a dedupe-match could over-ride
  60    * external identifier.
  61    *
  62    * @var bool
  63    */
  64   protected $_updateWithId;
  65   protected $_retCode;
  66
  67   protected $_externalIdentifierIndex;
  68   protected $_allExternalIdentifiers;
  69   protected $_parseStreetAddress;
  70
  71   /**
  72    * Array of successfully imported contact id's
  73    *
  74    * @var array
  75    */
  76   protected $_newContacts;
  77
  78   /**
  79    * Line count id.
  80    *
  81    * @var int
  82    */
  83   protected $_lineCount;
  84
  85   /**
  86    * Array of successfully imported related contact id's
  87    *
  88    * @var array
  89    */
  90   protected $_newRelatedContacts;
  91
  92   /**
  93    * Array of all the contacts whose street addresses are not parsed.
  94    * of this import process
  95    * @var array
  96    */
  97   protected $_unparsedStreetAddressContacts;
  98
  99   protected $_tableName;
 100
 101   /**
 102    * Total number of lines in file
 103    *
 104    * @var int
 105    */
 106   protected $_rowCount;
 107
 108   /**
 109    * Running total number of un-matched Contacts.
 110    *
 111    * @var int
 112    */
 113   protected $_unMatchCount;
 114
 115   /**
 116    * Array of unmatched lines.
 117    *
 118    * @var array
 119    */
 120   protected $_unMatch;
 121
 122   /**
 123    * Total number of contacts with unparsed addresses
 124    * @var int
 125    */
 126   protected $_unparsedAddressCount;
 127
 128   /**
 129    * Filename of mismatch data
 130    *
 131    * @var string
 132    */
 133   protected $_misMatchFilemName;
 134
 135   protected $_primaryKeyName;
 136   protected $_statusFieldName;
 137
 138   protected $fieldMetadata = [];
 139
 140   /**
 141    * Relationship labels.
 142    *
 143    * Temporary cache of labels to reduce queries in getRelationshipLabels.
 144    *
 145    * @var array
 146    *   e.g ['5a_b' => 'Employer', '5b_a' => 'Employee']
 147    */
 148   protected $relationshipLabels = [];
 149
 150   /**
 151    * On duplicate
 152    *
 153    * @var int
 154    */
 155   public $_onDuplicate;
 156
 157   /**
 158    * Dedupe rule group id to use if set
 159    *
 160    * @var int
 161    */
 162   public $_dedupeRuleGroupID = NULL;
 163
 164   /**
 165    * Class constructor.
 166    *
 167    * @param array $mapperKeys
 168    * @param array $mapperLocType
 169    * @param array $mapperPhoneType
 170    * @param array $mapperImProvider
 171    * @param array $mapperRelated
 172    * @param array $mapperRelatedContactType
 173    * @param array $mapperRelatedContactDetails
 174    * @param array $mapperRelatedContactLocType
 175    * @param array $mapperRelatedContactPhoneType
 176    * @param array $mapperRelatedContactImProvider
 177    * @param array $mapperWebsiteType
 178    * @param array $mapperRelatedContactWebsiteType
 179    */
 180   public function __construct(
 181     $mapperKeys = [], $mapperLocType = [], $mapperPhoneType = [], $mapperImProvider = [], $mapperRelated = [], $mapperRelatedContactType = [], $mapperRelatedContactDetails = [], $mapperRelatedContactLocType = [], $mapperRelatedContactPhoneType = [], $mapperRelatedContactImProvider = [],
 182     $mapperWebsiteType = [], $mapperRelatedContactWebsiteType = []
 183   ) {
 184     parent::__construct();
 185     $this->_mapperKeys = $mapperKeys;
 186     $this->_mapperLocType = &$mapperLocType;
 187     $this->_mapperPhoneType = &$mapperPhoneType;
 188     $this->_mapperWebsiteType = $mapperWebsiteType;
 189     // get IM service provider type id for contact
 190     $this->_mapperImProvider = &$mapperImProvider;
 191     $this->_mapperRelated = &$mapperRelated;
 192     $this->_mapperRelatedContactType = &$mapperRelatedContactType;
 193     $this->_mapperRelatedContactDetails = &$mapperRelatedContactDetails;
 194     $this->_mapperRelatedContactLocType = &$mapperRelatedContactLocType;
 195     $this->_mapperRelatedContactPhoneType = &$mapperRelatedContactPhoneType;
 196     $this->_mapperRelatedContactWebsiteType = $mapperRelatedContactWebsiteType;
 197     // get IM service provider type id for related contact
 198     $this->_mapperRelatedContactImProvider = &$mapperRelatedContactImProvider;
 199   }
 200
 201   /**
 202    * The initializer code, called before processing.
 203    */
 204   public function init() {
 205     $this->setFieldMetadata();
 206     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 207       $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field));
 208     }
 209     $this->_newContacts = [];
 210
 211     $this->setActiveFields($this->_mapperKeys);
 212     $this->setActiveFieldLocationTypes($this->_mapperLocType);
 213     $this->setActiveFieldPhoneTypes($this->_mapperPhoneType);
 214     $this->setActiveFieldWebsiteTypes($this->_mapperWebsiteType);
 215     //set active fields of IM provider of contact
 216     $this->setActiveFieldImProviders($this->_mapperImProvider);
 217
 218     //related info
 219     $this->setActiveFieldRelated($this->_mapperRelated);
 220     $this->setActiveFieldRelatedContactType($this->_mapperRelatedContactType);
 221     $this->setActiveFieldRelatedContactDetails($this->_mapperRelatedContactDetails);
 222     $this->setActiveFieldRelatedContactLocType($this->_mapperRelatedContactLocType);
 223     $this->setActiveFieldRelatedContactPhoneType($this->_mapperRelatedContactPhoneType);
 224     $this->setActiveFieldRelatedContactWebsiteType($this->_mapperRelatedContactWebsiteType);
 225     //set active fields of IM provider of related contact
 226     $this->setActiveFieldRelatedContactImProvider($this->_mapperRelatedContactImProvider);
 227
 228     $this->_phoneIndex = -1;
 229     $this->_emailIndex = -1;
 230     $this->_firstNameIndex = -1;
 231     $this->_lastNameIndex = -1;
 232     $this->_householdNameIndex = -1;
 233     $this->_organizationNameIndex = -1;
 234     $this->_externalIdentifierIndex = -1;
 235
 236     $index = 0;
 237     foreach ($this->_mapperKeys as $key) {
 238       if (substr($key, 0, 5) == 'email' && substr($key, 0, 14) != 'email_greeting') {
 239         $this->_emailIndex = $index;
 240         $this->_allEmails = [];
 241       }
 242       if (substr($key, 0, 5) == 'phone') {
 243         $this->_phoneIndex = $index;
 244       }
 245       if ($key == 'first_name') {
 246         $this->_firstNameIndex = $index;
 247       }
 248       if ($key == 'last_name') {
 249         $this->_lastNameIndex = $index;
 250       }
 251       if ($key == 'household_name') {
 252         $this->_householdNameIndex = $index;
 253       }
 254       if ($key == 'organization_name') {
 255         $this->_organizationNameIndex = $index;
 256       }
 257
 258       if ($key == 'external_identifier') {
 259         $this->_externalIdentifierIndex = $index;
 260         $this->_allExternalIdentifiers = [];
 261       }
 262       $index++;
 263     }
 264
 265     $this->_updateWithId = FALSE;
 266     if (in_array('id', $this->_mapperKeys) || ($this->_externalIdentifierIndex >= 0 && in_array($this->_onDuplicate, [
 267       CRM_Import_Parser::DUPLICATE_UPDATE,
 268       CRM_Import_Parser::DUPLICATE_FILL,
 269     ]))) {
 270       $this->_updateWithId = TRUE;
 271     }
 272
 273     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
 274   }
 275
 276   /**
 277    * Gets the fields available for importing in a key-name, title format.
 278    *
 279    * @return array
 280    *   eg. ['first_name' => 'First Name'.....]
 281    *
 282    * @throws \API_Exception
 283    *
 284    * @todo - we are constructing the metadata before we
 285    * have set the contact type so we re-do it here.
 286    *
 287    * Once we have cleaned up the way the mapper is handled
 288    * we can ditch all the existing _construct parameters in favour
 289    * of just the userJobID - there are current open PRs towards this end.
 290    */
 291   public function getAvailableFields(): array {
 292     $this->setFieldMetadata();
 293     $return = [];
 294     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
 295       if ($name === 'id' && $this->isSkipDuplicates()) {
 296         // Duplicates are being skipped so id matching is not availble.
 297         continue;
 298       }
 299       $return[$name] = $field['title'];
 300     }
 301     return $return;
 302   }
 303
 304   /**
 305    * Did the user specify duplicates should be skipped and not imported.
 306    *
 307    * @return bool
 308    *
 309    * @throws \API_Exception
 310    */
 311   private function isSkipDuplicates(): bool {
 312     return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP;
 313   }
 314
 315   /**
 316    * Handle the values in preview mode.
 317    *
 318    * @param array $values
 319    *   The array of values belonging to this line.
 320    *
 321    * @return bool
 322    *   the result of this processing
 323    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 324    */
 325   public function preview(&$values) {
 326     return $this->summary($values);
 327   }
 328
 329   /**
 330    * Handle the values in summary mode.
 331    *
 332    * @param array $values
 333    *   The array of values belonging to this line.
 334    *
 335    * @return bool
 336    *   the result of this processing
 337    *   CRM_Import_Parser::ERROR or CRM_Import_Parser::VALID
 338    */
 339   public function summary(&$values): int {
 340     $this->setActiveFieldValues($values);
 341     $rowNumber = (int) ($values[count($values) - 1]);
 342     $errorMessage = NULL;
 343     $errorRequired = FALSE;
 344     switch ($this->_contactType) {
 345       case 'Individual':
 346         $missingNames = [];
 347         if ($this->_firstNameIndex < 0 || empty($values[$this->_firstNameIndex])) {
 348           $errorRequired = TRUE;
 349           $missingNames[] = ts('First Name');
 350         }
 351         if ($this->_lastNameIndex < 0 || empty($values[$this->_lastNameIndex])) {
 352           $errorRequired = TRUE;
 353           $missingNames[] = ts('Last Name');
 354         }
 355         if ($errorRequired) {
 356           $and = ' ' . ts('and') . ' ';
 357           $errorMessage = ts('Missing required fields:') . ' ' . implode($and, $missingNames);
 358         }
 359         break;
 360
 361       case 'Household':
 362         if ($this->_householdNameIndex < 0 || empty($values[$this->_householdNameIndex])) {
 363           $errorRequired = TRUE;
 364           $errorMessage = ts('Missing required fields:') . ' ' . ts('Household Name');
 365         }
 366         break;
 367
 368       case 'Organization':
 369         if ($this->_organizationNameIndex < 0 || empty($values[$this->_organizationNameIndex])) {
 370           $errorRequired = TRUE;
 371           $errorMessage = ts('Missing required fields:') . ' ' . ts('Organization Name');
 372         }
 373         break;
 374     }
 375
 376     if ($this->_emailIndex >= 0) {
 377       /* If we don't have the required fields, bail */
 378
 379       if ($this->_contactType === 'Individual' && !$this->_updateWithId) {
 380         if ($errorRequired && empty($values[$this->_emailIndex])) {
 381           if ($errorMessage) {
 382             $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
 383           }
 384           else {
 385             $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
 386           }
 387           array_unshift($values, $errorMessage);
 388           $this->setImportStatus($rowNumber, 'ERROR', $errorMessage);
 389
 390           return CRM_Import_Parser::ERROR;
 391         }
 392       }
 393
 394       $email = $values[$this->_emailIndex] ?? NULL;
 395       if ($email) {
 396         /* If the email address isn't valid, bail */
 397
 398         if (!CRM_Utils_Rule::email($email)) {
 399           $errorMessage = ts('Invalid Email address');
 400           array_unshift($values, $errorMessage);
 401           $this->setImportStatus($rowNumber, 'ERROR', $errorMessage);
 402
 403           return CRM_Import_Parser::ERROR;
 404         }
 405
 406         /* otherwise, count it and move on */
 407         $this->_allEmails[$email] = $this->_lineCount;
 408       }
 409     }
 410     elseif ($errorRequired && !$this->_updateWithId) {
 411       if ($errorMessage) {
 412         $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
 413       }
 414       else {
 415         $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
 416       }
 417       array_unshift($values, $errorMessage);
 418       $this->setImportStatus($rowNumber, 'ERROR', $errorMessage);
 419
 420       return CRM_Import_Parser::ERROR;
 421     }
 422
 423     //check for duplicate external Identifier
 424     $externalID = $values[$this->_externalIdentifierIndex] ?? NULL;
 425     if ($externalID) {
 426       /* If it's a dupe,external Identifier  */
 427
 428       if ($externalDupe = CRM_Utils_Array::value($externalID, $this->_allExternalIdentifiers)) {
 429         $errorMessage = ts('External ID conflicts with record %1', [1 => $externalDupe]);
 430         array_unshift($values, $errorMessage);
 431         $this->setImportStatus($rowNumber, 'ERROR', $errorMessage);
 432         return CRM_Import_Parser::ERROR;
 433       }
 434       //otherwise, count it and move on
 435       $this->_allExternalIdentifiers[$externalID] = $this->_lineCount;
 436     }
 437
 438     //Checking error in custom data
 439     $params = &$this->getActiveFieldParams();
 440     $params['contact_type'] = $this->_contactType;
 441     //date-format part ends
 442
 443     $errorMessage = NULL;
 444
 445     //CRM-5125
 446     //add custom fields for contact sub type
 447     $csType = NULL;
 448     if (!empty($this->_contactSubType)) {
 449       $csType = $this->_contactSubType;
 450     }
 451
 452     //checking error in custom data
 453     $this->isErrorInCustomData($params, $errorMessage, $csType, $this->_relationships);
 454
 455     //checking error in core data
 456     $this->isErrorInCoreData($params, $errorMessage);
 457     if ($errorMessage) {
 458       $tempMsg = "Invalid value for field(s) : $errorMessage";
 459       $this->setImportStatus($rowNumber, 'ERROR', $tempMsg);
 460       array_unshift($values, $tempMsg);
 461       $errorMessage = NULL;
 462       return CRM_Import_Parser::ERROR;
 463     }
 464     $this->setImportStatus($rowNumber, 'NEW', '');
 465
 466     return CRM_Import_Parser::VALID;
 467   }
 468
 469   /**
 470    * Get Array of all the fields that could potentially be part
 471    * import process
 472    *
 473    * @return array
 474    */
 475   public function getAllFields() {
 476     return $this->_fields;
 477   }
 478
 479   /**
 480    * Handle the values in import mode.
 481    *
 482    * @param int $onDuplicate
 483    *   The code for what action to take on duplicates.
 484    * @param array $values
 485    *   The array of values belonging to this line.
 486    *
 487    * @param bool $doGeocodeAddress
 488    *
 489    * @return bool
 490    *   the result of this processing
 491    *
 492    * @throws \CiviCRM_API3_Exception
 493    * @throws \CRM_Core_Exception
 494    * @throws \API_Exception
 495    */
 496   public function import($onDuplicate, &$values, $doGeocodeAddress = FALSE) {
 497     $this->_unparsedStreetAddressContacts = [];
 498     if (!$doGeocodeAddress) {
 499       // CRM-5854, reset the geocode method to null to prevent geocoding
 500       CRM_Utils_GeocodeProvider::disableForSession();
 501     }
 502
 503     // first make sure this is a valid line
 504     //$this->_updateWithId = false;
 505     $response = $this->summary($values);
 506     $statusFieldName = $this->_statusFieldName;
 507
 508     if ($response != CRM_Import_Parser::VALID) {
 509       $this->setImportStatus((int) $values[count($values) - 1], 'Invalid', "Invalid (Error Code: $response)");
 510       return $response;
 511     }
 512
 513     $params = &$this->getActiveFieldParams();
 514     $formatted = [
 515       'contact_type' => $this->_contactType,
 516     ];
 517
 518     $contactFields = CRM_Contact_DAO_Contact::import();
 519
 520     //check if external identifier exists in database
 521     if (!empty($params['external_identifier']) && (!empty($params['id']) || in_array($onDuplicate, [
 522       CRM_Import_Parser::DUPLICATE_SKIP,
 523       CRM_Import_Parser::DUPLICATE_NOCHECK,
 524     ]))) {
 525
 526       $extIDResult = civicrm_api3('Contact', 'get', [
 527         'external_identifier' => $params['external_identifier'],
 528         'showAll' => 'all',
 529         'return' => ['id', 'contact_is_deleted'],
 530       ]);
 531       if (isset($extIDResult['id'])) {
 532         // record with matching external identifier does exist.
 533         $internalCid = $extIDResult['id'];
 534         if ($internalCid != CRM_Utils_Array::value('id', $params)) {
 535           if ($extIDResult['values'][$internalCid]['contact_is_deleted'] == 1) {
 536             // And it is deleted. What to do? If we skip it, they user
 537             // will be under the impression that the record exists in
 538             // the database, yet they won't be able to find it. If we
 539             // don't skip it, the database will try to insert a new record
 540             // with an external_identifier that is non-unique. So...
 541             // we will update this contact to remove the external_identifier
 542             // and let a new record be created.
 543             $update_params = ['id' => $internalCid, 'external_identifier' => ''];
 544             civicrm_api3('Contact', 'create', $update_params);
 545           }
 546           else {
 547             $errorMessage = ts('External ID already exists in Database.');
 548             array_unshift($values, $errorMessage);
 549             $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 550             return CRM_Import_Parser::DUPLICATE;
 551           }
 552         }
 553       }
 554     }
 555
 556     if (!empty($this->_contactSubType)) {
 557       $params['contact_sub_type'] = $this->_contactSubType;
 558     }
 559
 560     if ($subType = CRM_Utils_Array::value('contact_sub_type', $params)) {
 561       if (CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType, FALSE, 'label')) {
 562         $subTypes = CRM_Contact_BAO_ContactType::subTypePairs($this->_contactType, FALSE, NULL);
 563         $params['contact_sub_type'] = array_search($subType, $subTypes);
 564       }
 565       elseif (!CRM_Contact_BAO_ContactType::isExtendsContactType($subType, $this->_contactType)) {
 566         $message = "Mismatched or Invalid Contact Subtype.";
 567         array_unshift($values, $message);
 568         return CRM_Import_Parser::NO_MATCH;
 569       }
 570     }
 571
 572     // Get contact id to format common data in update/fill mode,
 573     // prioritising a dedupe rule check over an external_identifier check, but falling back on ext id.
 574     if ($this->_updateWithId && empty($params['id'])) {
 575       try {
 576         $possibleMatches = $this->getPossibleContactMatches($params);
 577       }
 578       catch (CRM_Core_Exception $e) {
 579         $errorMessage = $e->getMessage();
 580         array_unshift($values, $errorMessage);
 581
 582         $importRecordParams = [
 583           $statusFieldName => 'ERROR',
 584           "${statusFieldName}Msg" => $errorMessage,
 585         ];
 586         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 587         return CRM_Import_Parser::ERROR;
 588       }
 589       foreach ($possibleMatches as $possibleID) {
 590         $params['id'] = $formatted['id'] = $possibleID;
 591       }
 592     }
 593     //format common data, CRM-4062
 594     $this->formatCommonData($params, $formatted, $contactFields);
 595
 596     $relationship = FALSE;
 597     $createNewContact = TRUE;
 598     // Support Match and Update Via Contact ID
 599     if ($this->_updateWithId && isset($params['id'])) {
 600       $createNewContact = FALSE;
 601       // @todo - it feels like all the rows from here to the end of the IF
 602       // could be removed in favour of a simple check for whether the contact_type & id match
 603       $matchedIDs = $this->getIdsOfMatchingContacts($formatted);
 604       if (!empty($matchedIDs)) {
 605         if (count($matchedIDs) >= 1) {
 606           $updateflag = TRUE;
 607           foreach ($matchedIDs as $contactId) {
 608             if ($params['id'] == $contactId) {
 609               $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 610               if ($formatted['contact_type'] == $contactType) {
 611                 //validation of subtype for update mode
 612                 //CRM-5125
 613                 $contactSubType = NULL;
 614                 if (!empty($params['contact_sub_type'])) {
 615                   $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 616                 }
 617
 618                 if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 619
 620                   $message = "Mismatched contact SubTypes :";
 621                   array_unshift($values, $message);
 622                   $updateflag = FALSE;
 623                   $this->_retCode = CRM_Import_Parser::NO_MATCH;
 624                 }
 625                 else {
 626                   $updateflag = FALSE;
 627                   $this->_retCode = CRM_Import_Parser::VALID;
 628                 }
 629               }
 630               else {
 631                 $message = "Mismatched contact Types :";
 632                 array_unshift($values, $message);
 633                 $updateflag = FALSE;
 634                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 635               }
 636             }
 637           }
 638           if ($updateflag) {
 639             $message = "Mismatched contact IDs OR Mismatched contact Types :";
 640             array_unshift($values, $message);
 641             $this->_retCode = CRM_Import_Parser::NO_MATCH;
 642           }
 643         }
 644       }
 645       else {
 646         $contactType = NULL;
 647         if (!empty($params['id'])) {
 648           $contactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_type');
 649           if ($contactType) {
 650             if ($formatted['contact_type'] == $contactType) {
 651               //validation of subtype for update mode
 652               //CRM-5125
 653               $contactSubType = NULL;
 654               if (!empty($params['contact_sub_type'])) {
 655                 $contactSubType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params['id'], 'contact_sub_type');
 656               }
 657
 658               if (!empty($contactSubType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params['id'], $contactSubType) && $contactSubType != CRM_Utils_Array::value('contact_sub_type', $formatted))) {
 659
 660                 $message = "Mismatched contact SubTypes :";
 661                 array_unshift($values, $message);
 662                 $this->_retCode = CRM_Import_Parser::NO_MATCH;
 663               }
 664               else {
 665                 $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $params['id'], FALSE, $this->_dedupeRuleGroupID);
 666                 $this->_retCode = CRM_Import_Parser::VALID;
 667               }
 668             }
 669             else {
 670               $message = "Mismatched contact Types :";
 671               array_unshift($values, $message);
 672               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 673             }
 674           }
 675           else {
 676             // we should avoid multiple errors for single record
 677             // since we have already retCode and we trying to force again.
 678             if ($this->_retCode != CRM_Import_Parser::NO_MATCH) {
 679               $message = "No contact found for this contact ID:" . $params['id'];
 680               array_unshift($values, $message);
 681               $this->_retCode = CRM_Import_Parser::NO_MATCH;
 682             }
 683           }
 684         }
 685         else {
 686           //CRM-4148
 687           //now we want to create new contact on update/fill also.
 688           $createNewContact = TRUE;
 689         }
 690       }
 691
 692       if (isset($newContact) && is_a($newContact, 'CRM_Contact_BAO_Contact')) {
 693         $relationship = TRUE;
 694       }
 695     }
 696
 697     //fixed CRM-4148
 698     //now we create new contact in update/fill mode also.
 699     $contactID = NULL;
 700     if ($createNewContact || ($this->_retCode != CRM_Import_Parser::NO_MATCH && $this->_updateWithId)) {
 701       // @todo - there are multiple places where formatting is done that need consolidation.
 702       // This handles where the label has been passed in and it has gotten this far.
 703       // probably a bunch of hard-coded stuff could be removed to rely on this.
 704       $fields = Contact::getFields(FALSE)
 705         ->addWhere('options', '=', TRUE)
 706         ->setLoadOptions(TRUE)
 707         ->execute()->indexBy('name');
 708       foreach ($fields as $fieldName => $fieldSpec) {
 709         if (isset($formatted[$fieldName]) && is_array($formatted[$fieldName])) {
 710           // If we have an array at this stage, it's probably a multi-select
 711           // field that has already been parsed properly into the value that
 712           // should be inserted into the database.
 713           continue;
 714         }
 715         if (!empty($formatted[$fieldName])
 716           && empty($fieldSpec['options'][$formatted[$fieldName]])) {
 717           $formatted[$fieldName] = array_search($formatted[$fieldName], $fieldSpec['options'], TRUE) ?? $formatted[$fieldName];
 718         }
 719       }
 720       //CRM-4430, don't carry if not submitted.
 721       if ($this->_updateWithId && !empty($params['id'])) {
 722         $contactID = $params['id'];
 723       }
 724       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactID, TRUE, $this->_dedupeRuleGroupID);
 725     }
 726
 727     if (isset($newContact) && is_object($newContact) && ($newContact instanceof CRM_Contact_BAO_Contact)) {
 728       $relationship = TRUE;
 729       $newContact = clone($newContact);
 730       $contactID = $newContact->id;
 731       $this->_newContacts[] = $contactID;
 732
 733       //get return code if we create new contact in update mode, CRM-4148
 734       if ($this->_updateWithId) {
 735         $this->_retCode = CRM_Import_Parser::VALID;
 736       }
 737     }
 738     elseif (isset($newContact) && CRM_Core_Error::isAPIError($newContact, CRM_Core_Error::DUPLICATE_CONTACT)) {
 739       // if duplicate, no need of further processing
 740       if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
 741         $errorMessage = "Skipping duplicate record";
 742         array_unshift($values, $errorMessage);
 743         $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', $errorMessage);
 744         return CRM_Import_Parser::DUPLICATE;
 745       }
 746
 747       $relationship = TRUE;
 748       // CRM-10433/CRM-20739 - IDs could be string or array; handle accordingly
 749       if (!is_array($dupeContactIDs = $newContact['error_message']['params'][0])) {
 750         $dupeContactIDs = explode(',', $dupeContactIDs);
 751       }
 752       $dupeCount = count($dupeContactIDs);
 753       $contactID = array_pop($dupeContactIDs);
 754       // check to see if we had more than one duplicate contact id.
 755       // if we have more than one, the record will be rejected below
 756       if ($dupeCount == 1) {
 757         // there was only one dupe, we will continue normally...
 758         if (!in_array($contactID, $this->_newContacts)) {
 759           $this->_newContacts[] = $contactID;
 760         }
 761       }
 762     }
 763
 764     if ($contactID) {
 765       // call import hook
 766       $currentImportID = end($values);
 767
 768       $hookParams = [
 769         'contactID' => $contactID,
 770         'importID' => $currentImportID,
 771         'importTempTable' => $this->_tableName,
 772         'fieldHeaders' => $this->_mapperKeys,
 773         'fields' => $this->_activeFields,
 774       ];
 775
 776       CRM_Utils_Hook::import('Contact', 'process', $this, $hookParams);
 777     }
 778
 779     if ($relationship) {
 780       $primaryContactId = NULL;
 781       if (CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 782         if ($dupeCount == 1 && CRM_Utils_Rule::integer($contactID)) {
 783           $primaryContactId = $contactID;
 784         }
 785       }
 786       else {
 787         $primaryContactId = $newContact->id;
 788       }
 789
 790       if ((CRM_Core_Error::isAPIError($newContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || is_a($newContact, 'CRM_Contact_BAO_Contact')) && $primaryContactId) {
 791
 792         //relationship contact insert
 793         foreach ($params as $key => $field) {
 794           [$id, $first, $second] = CRM_Utils_System::explode('_', $key, 3);
 795           if (!($first == 'a' && $second == 'b') && !($first == 'b' && $second == 'a')) {
 796             continue;
 797           }
 798
 799           $relationType = new CRM_Contact_DAO_RelationshipType();
 800           $relationType->id = $id;
 801           $relationType->find(TRUE);
 802           $direction = "contact_sub_type_$second";
 803
 804           $formatting = [
 805             'contact_type' => $params[$key]['contact_type'],
 806           ];
 807
 808           //set subtype for related contact CRM-5125
 809           if (isset($relationType->$direction)) {
 810             //validation of related contact subtype for update mode
 811             if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $params[$key]) && $relCsType != $relationType->$direction) {
 812               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 813               array_unshift($values, $errorMessage);
 814               return CRM_Import_Parser::NO_MATCH;
 815             }
 816             else {
 817               $formatting['contact_sub_type'] = $relationType->$direction;
 818             }
 819           }
 820
 821           $contactFields = NULL;
 822           $contactFields = CRM_Contact_DAO_Contact::import();
 823
 824           //Relation on the basis of External Identifier.
 825           if (empty($params[$key]['id']) && !empty($params[$key]['external_identifier'])) {
 826             $params[$key]['id'] = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['external_identifier'], 'id', 'external_identifier');
 827           }
 828           // check for valid related contact id in update/fill mode, CRM-4424
 829           if (in_array($onDuplicate, [
 830             CRM_Import_Parser::DUPLICATE_UPDATE,
 831             CRM_Import_Parser::DUPLICATE_FILL,
 832           ]) && !empty($params[$key]['id'])) {
 833             $relatedContactType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_type');
 834             if (!$relatedContactType) {
 835               $errorMessage = ts("No contact found for this related contact ID: %1", [1 => $params[$key]['id']]);
 836               array_unshift($values, $errorMessage);
 837               return CRM_Import_Parser::NO_MATCH;
 838             }
 839
 840             //validation of related contact subtype for update mode
 841             //CRM-5125
 842             $relatedCsType = NULL;
 843             if (!empty($formatting['contact_sub_type'])) {
 844               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $params[$key]['id'], 'contact_sub_type');
 845             }
 846
 847             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($params[$key]['id'], $relatedCsType) &&
 848                 $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))
 849             ) {
 850               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.") . ' ' . ts("ID: %1", [1 => $params[$key]['id']]);
 851               array_unshift($values, $errorMessage);
 852               return CRM_Import_Parser::NO_MATCH;
 853             }
 854             // get related contact id to format data in update/fill mode,
 855             //if external identifier is present, CRM-4423
 856             $formatting['id'] = $params[$key]['id'];
 857           }
 858
 859           //format common data, CRM-4062
 860           $this->formatCommonData($field, $formatting, $contactFields);
 861
 862           //do we have enough fields to create related contact.
 863           $allowToCreate = $this->checkRelatedContactFields($key, $formatting);
 864
 865           if (!$allowToCreate) {
 866             $errorMessage = ts('Related contact required fields are missing.');
 867             array_unshift($values, $errorMessage);
 868             return CRM_Import_Parser::NO_MATCH;
 869           }
 870
 871           //fixed for CRM-4148
 872           if (!empty($params[$key]['id'])) {
 873             $contact = [
 874               'contact_id' => $params[$key]['id'],
 875             ];
 876             $defaults = [];
 877             $relatedNewContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
 878           }
 879           else {
 880             $relatedNewContact = $this->createContact($formatting, $contactFields, $onDuplicate, NULL, FALSE);
 881           }
 882
 883           if (is_object($relatedNewContact) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 884             $relatedNewContact = clone($relatedNewContact);
 885           }
 886
 887           $matchedIDs = [];
 888           // To update/fill contact, get the matching contact Ids if duplicate contact found
 889           // otherwise get contact Id from object of related contact
 890           if (is_array($relatedNewContact) && civicrm_error($relatedNewContact)) {
 891             if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 892               $matchedIDs = $relatedNewContact['error_message']['params'][0];
 893               if (!is_array($matchedIDs)) {
 894                 $matchedIDs = explode(',', $matchedIDs);
 895               }
 896             }
 897             else {
 898               $errorMessage = $relatedNewContact['error_message'];
 899               array_unshift($values, $errorMessage);
 900               $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
 901               return CRM_Import_Parser::ERROR;
 902             }
 903           }
 904           else {
 905             $matchedIDs[] = $relatedNewContact->id;
 906           }
 907           // update/fill related contact after getting matching Contact Ids, CRM-4424
 908           if (in_array($onDuplicate, [
 909             CRM_Import_Parser::DUPLICATE_UPDATE,
 910             CRM_Import_Parser::DUPLICATE_FILL,
 911           ])) {
 912             //validation of related contact subtype for update mode
 913             //CRM-5125
 914             $relatedCsType = NULL;
 915             if (!empty($formatting['contact_sub_type'])) {
 916               $relatedCsType = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $matchedIDs[0], 'contact_sub_type');
 917             }
 918
 919             if (!empty($relatedCsType) && (!CRM_Contact_BAO_ContactType::isAllowEdit($matchedIDs[0], $relatedCsType) && $relatedCsType != CRM_Utils_Array::value('contact_sub_type', $formatting))) {
 920               $errorMessage = ts("Mismatched or Invalid contact subtype found for this related contact.");
 921               array_unshift($values, $errorMessage);
 922               return CRM_Import_Parser::NO_MATCH;
 923             }
 924             else {
 925               $updatedContact = $this->createContact($formatting, $contactFields, $onDuplicate, $matchedIDs[0]);
 926             }
 927           }
 928           static $relativeContact = [];
 929           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 930             if (count($matchedIDs) >= 1) {
 931               $relContactId = $matchedIDs[0];
 932               //add relative contact to count during update & fill mode.
 933               //logic to make count distinct by contact id.
 934               if ($this->_newRelatedContacts || !empty($relativeContact)) {
 935                 $reContact = array_keys($relativeContact, $relContactId);
 936
 937                 if (empty($reContact)) {
 938                   $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 939                 }
 940               }
 941               else {
 942                 $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 943               }
 944             }
 945           }
 946           else {
 947             $relContactId = $relatedNewContact->id;
 948             $this->_newRelatedContacts[] = $relativeContact[] = $relContactId;
 949           }
 950
 951           if (CRM_Core_Error::isAPIError($relatedNewContact, CRM_Core_ERROR::DUPLICATE_CONTACT) || ($relatedNewContact instanceof CRM_Contact_BAO_Contact)) {
 952             //fix for CRM-1993.Checks for duplicate related contacts
 953             if (count($matchedIDs) >= 1) {
 954               //if more than one duplicate contact
 955               //found, create relationship with first contact
 956               // now create the relationship record
 957               $relationParams = [
 958                 'relationship_type_id' => $key,
 959                 'contact_check' => [
 960                   $relContactId => 1,
 961                 ],
 962                 'is_active' => 1,
 963                 'skipRecentView' => TRUE,
 964               ];
 965
 966               // we only handle related contact success, we ignore failures for now
 967               // at some point wold be nice to have related counts as separate
 968               $relationIds = [
 969                 'contact' => $primaryContactId,
 970               ];
 971
 972               [$valid, $duplicate] = self::legacyCreateMultiple($relationParams, $relationIds);
 973
 974               if ($valid || $duplicate) {
 975                 $relationIds['contactTarget'] = $relContactId;
 976                 $action = ($duplicate) ? CRM_Core_Action::UPDATE : CRM_Core_Action::ADD;
 977                 CRM_Contact_BAO_Relationship::relatedMemberships($primaryContactId, $relationParams, $relationIds, $action);
 978               }
 979
 980               //handle current employer, CRM-3532
 981               if ($valid) {
 982                 $allRelationships = CRM_Core_PseudoConstant::relationshipType('name');
 983                 $relationshipTypeId = str_replace([
 984                   '_a_b',
 985                   '_b_a',
 986                 ], [
 987                   '',
 988                   '',
 989                 ], $key);
 990                 $relationshipType = str_replace($relationshipTypeId . '_', '', $key);
 991                 $orgId = $individualId = NULL;
 992                 if ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employee of') {
 993                   $orgId = $relContactId;
 994                   $individualId = $primaryContactId;
 995                 }
 996                 elseif ($allRelationships[$relationshipTypeId]["name_{$relationshipType}"] == 'Employer of') {
 997                   $orgId = $primaryContactId;
 998                   $individualId = $relContactId;
 999                 }
1000                 if ($orgId && $individualId) {
1001                   $currentEmpParams[$individualId] = $orgId;
1002                   CRM_Contact_BAO_Contact_Utils::setCurrentEmployer($currentEmpParams);
1003                 }
1004               }
1005             }
1006           }
1007         }
1008       }
1009     }
1010     if ($this->_updateWithId) {
1011       //return warning if street address is unparsed, CRM-5886
1012       return $this->processMessage($values, $statusFieldName, $this->_retCode);
1013     }
1014     //dupe checking
1015     if (is_array($newContact) && civicrm_error($newContact)) {
1016       $code = NULL;
1017
1018       if (($code = CRM_Utils_Array::value('code', $newContact['error_message'])) && ($code == CRM_Core_Error::DUPLICATE_CONTACT)) {
1019         return $this->handleDuplicateError($newContact, $statusFieldName, $values, $onDuplicate, $formatted, $contactFields);
1020       }
1021       // Not a dupe, so we had an error
1022       $errorMessage = $newContact['error_message'];
1023       array_unshift($values, $errorMessage);
1024       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
1025       return CRM_Import_Parser::ERROR;
1026
1027     }
1028     // sleep(3);
1029     return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID);
1030   }
1031
1032   /**
1033    * Only called from import now... plus one place outside of core & tests.
1034    *
1035    * @todo - deprecate more aggressively - will involve copying to the import
1036    * class, adding a deprecation notice here & removing from tests.
1037    *
1038    * Takes an associative array and creates a relationship object.
1039    *
1040    * @deprecated For single creates use the api instead (it's tested).
1041    * For multiple a new variant of this function needs to be written and migrated to as this is a bit
1042    * nasty
1043    *
1044    * @param array $params
1045    *   (reference ) an assoc array of name/value pairs.
1046    * @param array $ids
1047    *   The array that holds all the db ids.
1048    *   per http://wiki.civicrm.org/confluence/display/CRM/Database+layer
1049    *  "we are moving away from the $ids param "
1050    *
1051    * @return array
1052    * @throws \CRM_Core_Exception
1053    */
1054   private static function legacyCreateMultiple($params, $ids = []) {
1055     // clarify that the only key ever pass in the ids array is 'contact'
1056     // There is legacy handling for other keys but a universe search on
1057     // calls to this function (not supported to be called from outside core)
1058     // only returns 2 calls - one in CRM_Contact_Import_Parser_Contact
1059     // and the other in jma grant applications (CRM_Grant_Form_Grant_Confirm)
1060     // both only pass in contact as a key here.
1061     $contactID = $ids['contact'];
1062     unset($ids);
1063     // There is only ever one value passed in from the 2 places above that call
1064     // this - by clarifying here like this we can cleanup within this
1065     // function without having to do more universe searches.
1066     $relatedContactID = key($params['contact_check']);
1067
1068     // check if the relationship is valid between contacts.
1069     // step 1: check if the relationship is valid if not valid skip and keep the count
1070     // step 2: check the if two contacts already have a relationship if yes skip and keep the count
1071     // step 3: if valid relationship then add the relation and keep the count
1072
1073     // step 1
1074     [$contactFields['relationship_type_id'], $firstLetter, $secondLetter] = explode('_', $params['relationship_type_id']);
1075     $contactFields['contact_id_' . $firstLetter] = $contactID;
1076     $contactFields['contact_id_' . $secondLetter] = $relatedContactID;
1077     if (!CRM_Contact_BAO_Relationship::checkRelationshipType($contactFields['contact_id_a'], $contactFields['contact_id_b'],
1078       $contactFields['relationship_type_id'])) {
1079       return [0, 0];
1080     }
1081
1082     if (
1083       CRM_Contact_BAO_Relationship::checkDuplicateRelationship(
1084         $contactFields,
1085         $contactID,
1086         // step 2
1087         $relatedContactID
1088       )
1089     ) {
1090       return [0, 1];
1091     }
1092
1093     $singleInstanceParams = array_merge($params, $contactFields);
1094     CRM_Contact_BAO_Relationship::add($singleInstanceParams);
1095     return [1, 0];
1096   }
1097
1098   /**
1099    * Format common params data to proper format to store.
1100    *
1101    * @param array $params
1102    *   Contain record values.
1103    * @param array $formatted
1104    *   Array of formatted data.
1105    * @param array $contactFields
1106    *   Contact DAO fields.
1107    */
1108   private function formatCommonData($params, &$formatted, $contactFields) {
1109     $csType = [
1110       CRM_Utils_Array::value('contact_type', $formatted),
1111     ];
1112
1113     //CRM-5125
1114     //add custom fields for contact sub type
1115     if (!empty($this->_contactSubType)) {
1116       $csType = $this->_contactSubType;
1117     }
1118
1119     if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $formatted)) {
1120       $csType = $relCsType;
1121     }
1122
1123     $customFields = CRM_Core_BAO_CustomField::getFields($formatted['contact_type'], FALSE, FALSE, $csType);
1124
1125     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
1126     $customFields = $customFields + $addressCustomFields;
1127
1128     //if a Custom Email Greeting, Custom Postal Greeting or Custom Addressee is mapped, and no "Greeting / Addressee Type ID" is provided, then automatically set the type = Customized, CRM-4575
1129     $elements = [
1130       'email_greeting_custom' => 'email_greeting',
1131       'postal_greeting_custom' => 'postal_greeting',
1132       'addressee_custom' => 'addressee',
1133     ];
1134     foreach ($elements as $k => $v) {
1135       if (array_key_exists($k, $params) && !(array_key_exists($v, $params))) {
1136         $label = key(CRM_Core_OptionGroup::values($v, TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1137         $params[$v] = $label;
1138       }
1139     }
1140
1141     //format date first
1142     $session = CRM_Core_Session::singleton();
1143     $dateType = $session->get("dateTypes");
1144     foreach ($params as $key => $val) {
1145       $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1146       if ($customFieldID &&
1147         !array_key_exists($customFieldID, $addressCustomFields)
1148       ) {
1149         //we should not update Date to null, CRM-4062
1150         if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
1151           //CRM-21267
1152           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
1153         }
1154         elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
1155           if (empty($val) && !is_numeric($val) && $this->_onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) {
1156             //retain earlier value when Import mode is `Fill`
1157             unset($params[$key]);
1158           }
1159           else {
1160             $params[$key] = CRM_Utils_String::strtoboolstr($val);
1161           }
1162         }
1163       }
1164
1165       if ($key == 'birth_date' && $val) {
1166         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
1167       }
1168       elseif ($key == 'deceased_date' && $val) {
1169         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
1170         $params['is_deceased'] = 1;
1171       }
1172       elseif ($key == 'is_deceased' && $val) {
1173         $params[$key] = CRM_Utils_String::strtoboolstr($val);
1174       }
1175     }
1176
1177     //now format custom data.
1178     foreach ($params as $key => $field) {
1179       if (is_array($field)) {
1180         $isAddressCustomField = FALSE;
1181         foreach ($field as $value) {
1182           $break = FALSE;
1183           if (is_array($value)) {
1184             foreach ($value as $name => $testForEmpty) {
1185               if ($addressCustomFieldID = CRM_Core_BAO_CustomField::getKeyID($name)) {
1186                 $isAddressCustomField = TRUE;
1187                 break;
1188               }
1189               // check if $value does not contain IM provider or phoneType
1190               if (($name !== 'phone_type_id' || $name !== 'provider_id') && ($testForEmpty === '' || $testForEmpty == NULL)) {
1191                 $break = TRUE;
1192                 break;
1193               }
1194             }
1195           }
1196           else {
1197             $break = TRUE;
1198           }
1199
1200           if (!$break) {
1201             if (!empty($value['location_type_id'])) {
1202               $this->formatLocationBlock($value, $formatted);
1203             }
1204             else {
1205               // @todo - this is still reachable - e.g. import with related contact info like firstname,lastname,spouse-first-name,spouse-last-name,spouse-home-phone
1206               CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
1207               $this->formatContactParameters($value, $formatted);
1208             }
1209           }
1210         }
1211         if (!$isAddressCustomField) {
1212           continue;
1213         }
1214       }
1215
1216       $formatValues = [
1217         $key => $field,
1218       ];
1219
1220       if (($key !== 'preferred_communication_method') && (array_key_exists($key, $contactFields))) {
1221         // due to merging of individual table and
1222         // contact table, we need to avoid
1223         // preferred_communication_method forcefully
1224         $formatValues['contact_type'] = $formatted['contact_type'];
1225       }
1226
1227       if ($key == 'id' && isset($field)) {
1228         $formatted[$key] = $field;
1229       }
1230       $this->formatContactParameters($formatValues, $formatted);
1231
1232       //Handling Custom Data
1233       // note: Address custom fields will be handled separately inside formatContactParameters
1234       if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
1235         array_key_exists($customFieldID, $customFields) &&
1236         !array_key_exists($customFieldID, $addressCustomFields)
1237       ) {
1238
1239         $extends = $customFields[$customFieldID]['extends'] ?? NULL;
1240         $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1241         $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
1242         $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1243
1244         if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
1245           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1246           foreach ($customOption as $customValue) {
1247             $val = $customValue['value'] ?? NULL;
1248             $label = strtolower($customValue['label'] ?? '');
1249             $value = strtolower(trim($formatted[$key]));
1250             if (($value == $label) || ($value == strtolower($val))) {
1251               $params[$key] = $formatted[$key] = $val;
1252             }
1253           }
1254         }
1255         elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
1256           $mulValues = explode(',', $formatted[$key]);
1257           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1258           $formatted[$key] = [];
1259           $params[$key] = [];
1260           foreach ($mulValues as $v1) {
1261             foreach ($customOption as $v2) {
1262               if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1263                 (strtolower($v2['value']) == strtolower(trim($v1)))
1264               ) {
1265                 if ($htmlType == 'CheckBox') {
1266                   $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
1267                 }
1268                 else {
1269                   $params[$key][] = $formatted[$key][] = $v2['value'];
1270                 }
1271               }
1272             }
1273           }
1274         }
1275       }
1276     }
1277
1278     if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields) &&
1279       !array_key_exists($customFieldID, $addressCustomFields)
1280     ) {
1281       // @todo calling api functions directly is not supported
1282       _civicrm_api3_custom_format_params($params, $formatted, $extends);
1283     }
1284
1285     // to check if not update mode and unset the fields with empty value.
1286     if (!$this->_updateWithId && array_key_exists('custom', $formatted)) {
1287       foreach ($formatted['custom'] as $customKey => $customvalue) {
1288         if (empty($formatted['custom'][$customKey][-1]['is_required'])) {
1289           $formatted['custom'][$customKey][-1]['is_required'] = $customFields[$customKey]['is_required'];
1290         }
1291         $emptyValue = $customvalue[-1]['value'] ?? NULL;
1292         if (!isset($emptyValue)) {
1293           unset($formatted['custom'][$customKey]);
1294         }
1295       }
1296     }
1297
1298     // parse street address, CRM-5450
1299     if ($this->_parseStreetAddress) {
1300       if (array_key_exists('address', $formatted) && is_array($formatted['address'])) {
1301         foreach ($formatted['address'] as $instance => & $address) {
1302           $streetAddress = $address['street_address'] ?? NULL;
1303           if (empty($streetAddress)) {
1304             continue;
1305           }
1306           // parse address field.
1307           $parsedFields = CRM_Core_BAO_Address::parseStreetAddress($streetAddress);
1308
1309           //street address consider to be parsed properly,
1310           //If we get street_name and street_number.
1311           if (empty($parsedFields['street_name']) || empty($parsedFields['street_number'])) {
1312             $parsedFields = array_fill_keys(array_keys($parsedFields), '');
1313           }
1314
1315           // merge parse address w/ main address block.
1316           $address = array_merge($address, $parsedFields);
1317         }
1318       }
1319     }
1320   }
1321
1322   /**
1323    * Get the array of successfully imported contact id's
1324    *
1325    * @return array
1326    */
1327   public function getImportedContacts() {
1328     return $this->_newContacts;
1329   }
1330
1331   /**
1332    * Get the array of successfully imported related contact id's
1333    *
1334    * @return array
1335    */
1336   public function &getRelatedImportedContacts() {
1337     return $this->_newRelatedContacts;
1338   }
1339
1340   /**
1341    * Check if an error in custom data.
1342    *
1343    * @param array $params
1344    * @param string $errorMessage
1345    *   A string containing all the error-fields.
1346    *
1347    * @param null $csType
1348    * @param null $relationships
1349    */
1350   public static function isErrorInCustomData($params, &$errorMessage, $csType = NULL, $relationships = NULL) {
1351     $dateType = CRM_Core_Session::singleton()->get("dateTypes");
1352     $errors = [];
1353
1354     if (!empty($params['contact_sub_type'])) {
1355       $csType = $params['contact_sub_type'] ?? NULL;
1356     }
1357
1358     if (empty($params['contact_type'])) {
1359       $params['contact_type'] = 'Individual';
1360     }
1361
1362     // get array of subtypes - CRM-18708
1363     if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) {
1364       $csType = self::getSubtypes($params['contact_type']);
1365     }
1366
1367     if (is_array($csType)) {
1368       // fetch custom fields for every subtype and add it to $customFields array
1369       // CRM-18708
1370       $customFields = [];
1371       foreach ($csType as $cType) {
1372         $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType);
1373       }
1374     }
1375     else {
1376       $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType);
1377     }
1378
1379     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
1380     $customFields = $customFields + $addressCustomFields;
1381     foreach ($params as $key => $value) {
1382       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1383         /* check if it's a valid custom field id */
1384
1385         if (!array_key_exists($customFieldID, $customFields)) {
1386           $errors[] = ts('field ID');
1387         }
1388         // validate null values for required custom fields of type boolean
1389         if (!empty($customFields[$customFieldID]['is_required']) && (empty($params['custom_' . $customFieldID]) && !is_numeric($params['custom_' . $customFieldID])) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
1390           $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1391         }
1392
1393         //For address custom fields, we do get actual custom field value as an inner array of
1394         //values so need to modify
1395         if (array_key_exists($customFieldID, $addressCustomFields)) {
1396           $value = $value[0][$key];
1397         }
1398         /* validate the data against the CF type */
1399
1400         if ($value) {
1401           $dataType = $customFields[$customFieldID]['data_type'];
1402           $htmlType = $customFields[$customFieldID]['html_type'];
1403           $isSerialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
1404           if ($dataType == 'Date') {
1405             if (array_key_exists($customFieldID, $addressCustomFields) && CRM_Utils_Date::convertToDefaultDate($params[$key][0], $dateType, $key)) {
1406               $value = $params[$key][0][$key];
1407             }
1408             elseif (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1409               $value = $params[$key];
1410             }
1411             else {
1412               $errors[] = $customFields[$customFieldID]['label'];
1413             }
1414           }
1415           elseif ($dataType == 'Boolean') {
1416             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1417               $errors[] = $customFields[$customFieldID]['label'] . '::' . $customFields[$customFieldID]['groupTitle'];
1418             }
1419           }
1420           // need not check for label filed import
1421           $selectHtmlTypes = [
1422             'CheckBox',
1423             'Select',
1424             'Radio',
1425           ];
1426           if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') {
1427             $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value);
1428             if (!$valid) {
1429               $errors[] = $customFields[$customFieldID]['label'];
1430             }
1431           }
1432
1433           // check for values for custom fields for checkboxes and multiselect
1434           if ($isSerialized && $dataType != 'ContactReference') {
1435             $value = trim($value);
1436             $value = str_replace('|', ',', $value);
1437             $mulValues = explode(',', $value);
1438             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1439             foreach ($mulValues as $v1) {
1440               if (strlen($v1) == 0) {
1441                 continue;
1442               }
1443
1444               $flag = FALSE;
1445               foreach ($customOption as $v2) {
1446                 if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1447                   $flag = TRUE;
1448                 }
1449               }
1450
1451               if (!$flag) {
1452                 $errors[] = $customFields[$customFieldID]['label'];
1453               }
1454             }
1455           }
1456           elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) {
1457             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1458             $flag = FALSE;
1459             foreach ($customOption as $v2) {
1460               if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1461                 $flag = TRUE;
1462               }
1463             }
1464             if (!$flag) {
1465               $errors[] = $customFields[$customFieldID]['label'];
1466             }
1467           }
1468           elseif ($isSerialized && $dataType === 'StateProvince') {
1469             $mulValues = explode(',', $value);
1470             foreach ($mulValues as $stateValue) {
1471               if ($stateValue) {
1472                 if (self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvinceAbbreviation()) || self::in_value(trim($stateValue), CRM_Core_PseudoConstant::stateProvince())) {
1473                   continue;
1474                 }
1475                 else {
1476                   $errors[] = $customFields[$customFieldID]['label'];
1477                 }
1478               }
1479             }
1480           }
1481           elseif ($isSerialized && $dataType == 'Country') {
1482             $mulValues = explode(',', $value);
1483             foreach ($mulValues as $countryValue) {
1484               if ($countryValue) {
1485                 CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1486                 CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1487                 $limitCodes = CRM_Core_BAO_Country::countryLimit();
1488
1489                 $error = TRUE;
1490                 foreach ([
1491                   $countryNames,
1492                   $countryIsoCodes,
1493                   $limitCodes,
1494                 ] as $values) {
1495                   if (in_array(trim($countryValue), $values)) {
1496                     $error = FALSE;
1497                     break;
1498                   }
1499                 }
1500
1501                 if ($error) {
1502                   $errors[] = $customFields[$customFieldID]['label'];
1503                 }
1504               }
1505             }
1506           }
1507         }
1508       }
1509       elseif (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1510         //CRM-5125
1511         //supporting custom data of related contact subtypes
1512         $relation = NULL;
1513         if ($relationships) {
1514           if (array_key_exists($key, $relationships)) {
1515             $relation = $key;
1516           }
1517           elseif (CRM_Utils_Array::key($key, $relationships)) {
1518             $relation = CRM_Utils_Array::key($key, $relationships);
1519           }
1520         }
1521         if (!empty($relation)) {
1522           [$id, $first, $second] = CRM_Utils_System::explode('_', $relation, 3);
1523           $direction = "contact_sub_type_$second";
1524           $relationshipType = new CRM_Contact_BAO_RelationshipType();
1525           $relationshipType->id = $id;
1526           if ($relationshipType->find(TRUE)) {
1527             if (isset($relationshipType->$direction)) {
1528               $params[$key]['contact_sub_type'] = $relationshipType->$direction;
1529             }
1530           }
1531         }
1532
1533         self::isErrorInCustomData($params[$key], $errorMessage, $csType, $relationships);
1534       }
1535     }
1536     if ($errors) {
1537       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1538     }
1539   }
1540
1541   /**
1542    * Check if value present in all genders or.
1543    * as a substring of any gender value, if yes than return corresponding gender.
1544    * eg value might be  m/M, ma/MA, mal/MAL, male return 'Male'
1545    * but if value is 'maleabc' than return false
1546    *
1547    * @param string $gender
1548    *   Check this value across gender values.
1549    *
1550    * retunr gender value / false
1551    *
1552    * @return bool
1553    */
1554   public function checkGender($gender) {
1555     $gender = trim($gender, '.');
1556     if (!$gender) {
1557       return FALSE;
1558     }
1559
1560     $allGenders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
1561     foreach ($allGenders as $key => $value) {
1562       if (strlen($gender) > strlen($value)) {
1563         continue;
1564       }
1565       if ($gender == $value) {
1566         return $value;
1567       }
1568       if (substr_compare($value, $gender, 0, strlen($gender), TRUE) === 0) {
1569         return $value;
1570       }
1571     }
1572
1573     return FALSE;
1574   }
1575
1576   /**
1577    * Check if an error in Core( non-custom fields ) field
1578    *
1579    * @param array $params
1580    * @param string $errorMessage
1581    *   A string containing all the error-fields.
1582    */
1583   public function isErrorInCoreData($params, &$errorMessage) {
1584     $errors = [];
1585     foreach ($params as $key => $value) {
1586       if ($value) {
1587         $session = CRM_Core_Session::singleton();
1588         $dateType = $session->get("dateTypes");
1589
1590         switch ($key) {
1591           case 'birth_date':
1592             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1593               if (!CRM_Utils_Rule::date($params[$key])) {
1594                 $errors[] = ts('Birth Date');
1595               }
1596             }
1597             else {
1598               $errors[] = ts('Birth-Date');
1599             }
1600             break;
1601
1602           case 'deceased_date':
1603             if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) {
1604               if (!CRM_Utils_Rule::date($params[$key])) {
1605                 $errors[] = ts('Deceased Date');
1606               }
1607             }
1608             else {
1609               $errors[] = ts('Deceased Date');
1610             }
1611             break;
1612
1613           case 'is_deceased':
1614             if (CRM_Utils_String::strtoboolstr($value) === FALSE) {
1615               $errors[] = ts('Deceased');
1616             }
1617             break;
1618
1619           case 'gender_id':
1620             if (!self::checkGender($value)) {
1621               $errors[] = ts('Gender');
1622             }
1623             break;
1624
1625           case 'preferred_communication_method':
1626             $preffComm = [];
1627             $preffComm = explode(',', $value);
1628             foreach ($preffComm as $v) {
1629               if (!self::in_value(trim($v), CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method'))) {
1630                 $errors[] = ts('Preferred Communication Method');
1631               }
1632             }
1633             break;
1634
1635           case 'preferred_mail_format':
1636             if (!array_key_exists(strtolower($value), array_change_key_case(CRM_Core_SelectValues::pmf(), CASE_LOWER))) {
1637               $errors[] = ts('Preferred Mail Format');
1638             }
1639             break;
1640
1641           case 'individual_prefix':
1642           case 'prefix_id':
1643             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id'))) {
1644               $errors[] = ts('Individual Prefix');
1645             }
1646             break;
1647
1648           case 'individual_suffix':
1649           case 'suffix_id':
1650             if (!self::in_value($value, CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id'))) {
1651               $errors[] = ts('Individual Suffix');
1652             }
1653             break;
1654
1655           case 'state_province':
1656             if (!empty($value)) {
1657               foreach ($value as $stateValue) {
1658                 if ($stateValue['state_province']) {
1659                   if (self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvinceAbbreviation()) ||
1660                     self::in_value($stateValue['state_province'], CRM_Core_PseudoConstant::stateProvince())
1661                   ) {
1662                     continue;
1663                   }
1664                   else {
1665                     $errors[] = ts('State/Province');
1666                   }
1667                 }
1668               }
1669             }
1670             break;
1671
1672           case 'country':
1673             if (!empty($value)) {
1674               foreach ($value as $stateValue) {
1675                 if ($stateValue['country']) {
1676                   CRM_Core_PseudoConstant::populate($countryNames, 'CRM_Core_DAO_Country', TRUE, 'name', 'is_active');
1677                   CRM_Core_PseudoConstant::populate($countryIsoCodes, 'CRM_Core_DAO_Country', TRUE, 'iso_code');
1678                   $limitCodes = CRM_Core_BAO_Country::countryLimit();
1679                   //If no country is selected in
1680                   //localization then take all countries
1681                   if (empty($limitCodes)) {
1682                     $limitCodes = $countryIsoCodes;
1683                   }
1684
1685                   if (self::in_value($stateValue['country'], $limitCodes) || self::in_value($stateValue['country'], CRM_Core_PseudoConstant::country())) {
1686                     continue;
1687                   }
1688                   if (self::in_value($stateValue['country'], $countryIsoCodes) || self::in_value($stateValue['country'], $countryNames)) {
1689                     $errors[] = ts('Country input value is in table but not "available": "This Country is valid but is NOT in the list of Available Countries currently configured for your site. This can be viewed and modifed from Administer > Localization > Languages Currency Locations." ');
1690                   }
1691                   else {
1692                     $errors[] = ts('Country input value not in country table: "The Country value appears to be invalid. It does not match any value in CiviCRM table of countries."');
1693                   }
1694                 }
1695               }
1696             }
1697             break;
1698
1699           case 'county':
1700             if (!empty($value)) {
1701               foreach ($value as $county) {
1702                 if ($county['county']) {
1703                   $countyNames = CRM_Core_PseudoConstant::county();
1704                   if (!empty($county['county']) && !in_array($county['county'], $countyNames)) {
1705                     $errors[] = ts('County input value not in county table: The County value appears to be invalid. It does not match any value in CiviCRM table of counties.');
1706                   }
1707                 }
1708               }
1709             }
1710             break;
1711
1712           case 'geo_code_1':
1713             if (!empty($value)) {
1714               foreach ($value as $codeValue) {
1715                 if (!empty($codeValue['geo_code_1'])) {
1716                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_1'])) {
1717                     continue;
1718                   }
1719                   $errors[] = ts('Geo code 1');
1720                 }
1721               }
1722             }
1723             break;
1724
1725           case 'geo_code_2':
1726             if (!empty($value)) {
1727               foreach ($value as $codeValue) {
1728                 if (!empty($codeValue['geo_code_2'])) {
1729                   if (CRM_Utils_Rule::numeric($codeValue['geo_code_2'])) {
1730                     continue;
1731                   }
1732                   $errors[] = ts('Geo code 2');
1733                 }
1734               }
1735             }
1736             break;
1737
1738           //check for any error in email/postal greeting, addressee,
1739           //custom email/postal greeting, custom addressee, CRM-4575
1740
1741           case 'email_greeting':
1742             $emailGreetingFilter = [
1743               'contact_type' => $this->_contactType,
1744               'greeting_type' => 'email_greeting',
1745             ];
1746             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($emailGreetingFilter))) {
1747               $errors[] = ts('Email Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Email Greetings for valid values');
1748             }
1749             break;
1750
1751           case 'postal_greeting':
1752             $postalGreetingFilter = [
1753               'contact_type' => $this->_contactType,
1754               'greeting_type' => 'postal_greeting',
1755             ];
1756             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($postalGreetingFilter))) {
1757               $errors[] = ts('Postal Greeting must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Postal Greetings for valid values');
1758             }
1759             break;
1760
1761           case 'addressee':
1762             $addresseeFilter = [
1763               'contact_type' => $this->_contactType,
1764               'greeting_type' => 'addressee',
1765             ];
1766             if (!self::in_value($value, CRM_Core_PseudoConstant::greeting($addresseeFilter))) {
1767               $errors[] = ts('Addressee must be one of the configured format options. Check Administer >> System Settings >> Option Groups >> Addressee for valid values');
1768             }
1769             break;
1770
1771           case 'email_greeting_custom':
1772             if (array_key_exists('email_greeting', $params)) {
1773               $emailGreetingLabel = key(CRM_Core_OptionGroup::values('email_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1774               if (CRM_Utils_Array::value('email_greeting', $params) != $emailGreetingLabel) {
1775                 $errors[] = ts('Email Greeting - Custom');
1776               }
1777             }
1778             break;
1779
1780           case 'postal_greeting_custom':
1781             if (array_key_exists('postal_greeting', $params)) {
1782               $postalGreetingLabel = key(CRM_Core_OptionGroup::values('postal_greeting', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1783               if (CRM_Utils_Array::value('postal_greeting', $params) != $postalGreetingLabel) {
1784                 $errors[] = ts('Postal Greeting - Custom');
1785               }
1786             }
1787             break;
1788
1789           case 'addressee_custom':
1790             if (array_key_exists('addressee', $params)) {
1791               $addresseeLabel = key(CRM_Core_OptionGroup::values('addressee', TRUE, NULL, NULL, 'AND v.name = "Customized"'));
1792               if (CRM_Utils_Array::value('addressee', $params) != $addresseeLabel) {
1793                 $errors[] = ts('Addressee - Custom');
1794               }
1795             }
1796             break;
1797
1798           case 'url':
1799             if (is_array($value)) {
1800               foreach ($value as $values) {
1801                 if (!empty($values['url']) && !CRM_Utils_Rule::url($values['url'])) {
1802                   $errors[] = ts('Website');
1803                   break;
1804                 }
1805               }
1806             }
1807             break;
1808
1809           case 'do_not_email':
1810           case 'do_not_phone':
1811           case 'do_not_mail':
1812           case 'do_not_sms':
1813           case 'do_not_trade':
1814             if (CRM_Utils_Rule::boolean($value) == FALSE) {
1815               $key = ucwords(str_replace("_", " ", $key));
1816               $errors[] = $key;
1817             }
1818             break;
1819
1820           case 'email':
1821             if (is_array($value)) {
1822               foreach ($value as $values) {
1823                 if (!empty($values['email']) && !CRM_Utils_Rule::email($values['email'])) {
1824                   $errors[] = $key;
1825                   break;
1826                 }
1827               }
1828             }
1829             break;
1830
1831           default:
1832             if (is_array($params[$key]) && isset($params[$key]["contact_type"])) {
1833               //check for any relationship data ,FIX ME
1834               self::isErrorInCoreData($params[$key], $errorMessage);
1835             }
1836         }
1837       }
1838     }
1839     if ($errors) {
1840       $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', $errors);
1841     }
1842   }
1843
1844   /**
1845    * Ckeck a value present or not in a array.
1846    *
1847    * @param $value
1848    * @param $valueArray
1849    *
1850    * @return bool
1851    */
1852   public static function in_value($value, $valueArray) {
1853     foreach ($valueArray as $key => $v) {
1854       //fix for CRM-1514
1855       if (strtolower(trim($v, ".")) == strtolower(trim($value, "."))) {
1856         return TRUE;
1857       }
1858     }
1859     return FALSE;
1860   }
1861
1862   /**
1863    * Build error-message containing error-fields
1864    *
1865    * Once upon a time there was a dev who hadn't heard of implode. That dev wrote this function.
1866    *
1867    * @todo just say no!
1868    *
1869    * @param string $errorName
1870    *   A string containing error-field name.
1871    * @param string $errorMessage
1872    *   A string containing all the error-fields, where the new errorName is concatenated.
1873    *
1874    */
1875   public static function addToErrorMsg($errorName, &$errorMessage) {
1876     if ($errorMessage) {
1877       $errorMessage .= "; $errorName";
1878     }
1879     else {
1880       $errorMessage = $errorName;
1881     }
1882   }
1883
1884   /**
1885    * Method for creating contact.
1886    *
1887    * @param array $formatted
1888    * @param array $contactFields
1889    * @param int $onDuplicate
1890    * @param int $contactId
1891    * @param bool $requiredCheck
1892    * @param int $dedupeRuleGroupID
1893    *
1894    * @return array|bool|\CRM_Contact_BAO_Contact|\CRM_Core_Error|null
1895    */
1896   public function createContact(&$formatted, &$contactFields, $onDuplicate, $contactId = NULL, $requiredCheck = TRUE, $dedupeRuleGroupID = NULL) {
1897     $dupeCheck = FALSE;
1898     $newContact = NULL;
1899
1900     if (is_null($contactId) && ($onDuplicate != CRM_Import_Parser::DUPLICATE_NOCHECK)) {
1901       $dupeCheck = (bool) ($onDuplicate);
1902     }
1903
1904     //get the prefix id etc if exists
1905     CRM_Contact_BAO_Contact::resolveDefaults($formatted, TRUE);
1906
1907     //@todo direct call to API function not supported.
1908     // setting required check to false, CRM-2839
1909     // plus we do our own required check in import
1910     try {
1911       $error = $this->deprecated_contact_check_params($formatted, $dupeCheck, $dedupeRuleGroupID);
1912       if ($error) {
1913         return $error;
1914       }
1915       $this->deprecated_validate_formatted_contact($formatted);
1916     }
1917     catch (CRM_Core_Exception $e) {
1918       return ['error_message' => $e->getMessage(), 'is_error' => 1, 'code' => $e->getCode()];
1919     }
1920
1921     if ($contactId) {
1922       $this->formatParams($formatted, $onDuplicate, (int) $contactId);
1923     }
1924
1925     // Resetting and rebuilding cache could be expensive.
1926     CRM_Core_Config::setPermitCacheFlushMode(FALSE);
1927
1928     // If a user has logged in, or accessed via a checksum
1929     // Then deliberately 'blanking' a value in the profile should remove it from their record
1930     // @todo this should either be TRUE or FALSE in the context of import - once
1931     // we figure out which we can remove all the rest.
1932     // Also note the meaning of this parameter is less than it used to
1933     // be following block cleanup.
1934     $formatted['updateBlankLocInfo'] = TRUE;
1935     if ((CRM_Core_Session::singleton()->get('authSrc') & (CRM_Core_Permission::AUTH_SRC_CHECKSUM + CRM_Core_Permission::AUTH_SRC_LOGIN)) == 0) {
1936       $formatted['updateBlankLocInfo'] = FALSE;
1937     }
1938
1939     [$data, $contactDetails] = CRM_Contact_BAO_Contact::formatProfileContactParams($formatted, $contactFields, $contactId, NULL, $formatted['contact_type']);
1940
1941     // manage is_opt_out
1942     if (array_key_exists('is_opt_out', $contactFields) && array_key_exists('is_opt_out', $formatted)) {
1943       $wasOptOut = $contactDetails['is_opt_out'] ?? FALSE;
1944       $isOptOut = $formatted['is_opt_out'];
1945       $data['is_opt_out'] = $isOptOut;
1946       // on change, create new civicrm_subscription_history entry
1947       if (($wasOptOut != $isOptOut) && !empty($contactDetails['contact_id'])) {
1948         $shParams = [
1949           'contact_id' => $contactDetails['contact_id'],
1950           'status' => $isOptOut ? 'Removed' : 'Added',
1951           'method' => 'Web',
1952         ];
1953         CRM_Contact_BAO_SubscriptionHistory::create($shParams);
1954       }
1955     }
1956
1957     $contact = civicrm_api3('Contact', 'create', $data);
1958     $cid = $contact['id'];
1959
1960     CRM_Core_Config::setPermitCacheFlushMode(TRUE);
1961
1962     $contact = [
1963       'contact_id' => $cid,
1964     ];
1965
1966     $defaults = [];
1967     $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
1968
1969     //get the id of the contact whose street address is not parsable, CRM-5886
1970     if ($this->_parseStreetAddress && is_object($newContact) && property_exists($newContact, 'address') && $newContact->address) {
1971       foreach ($newContact->address as $address) {
1972         if (!empty($address['street_address']) && (empty($address['street_number']) || empty($address['street_name']))) {
1973           $this->_unparsedStreetAddressContacts[] = [
1974             'id' => $newContact->id,
1975             'streetAddress' => $address['street_address'],
1976           ];
1977         }
1978       }
1979     }
1980     return $newContact;
1981   }
1982
1983   /**
1984    * Format params for update and fill mode.
1985    *
1986    * @param array $params
1987    *   reference to an array containing all the.
1988    *   values for import
1989    * @param int $onDuplicate
1990    * @param int $cid
1991    *   contact id.
1992    */
1993   public function formatParams(&$params, $onDuplicate, $cid) {
1994     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
1995       return;
1996     }
1997
1998     $contactParams = [
1999       'contact_id' => $cid,
2000     ];
2001
2002     $defaults = [];
2003     $contactObj = CRM_Contact_BAO_Contact::retrieve($contactParams, $defaults);
2004
2005     $modeFill = ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL);
2006
2007     $groupTree = CRM_Core_BAO_CustomGroup::getTree($params['contact_type'], NULL, $cid, 0, NULL);
2008     CRM_Core_BAO_CustomGroup::setDefaults($groupTree, $defaults, FALSE, FALSE);
2009
2010     $locationFields = [
2011       'email' => 'email',
2012       'phone' => 'phone',
2013       'im' => 'name',
2014       'website' => 'website',
2015       'address' => 'address',
2016     ];
2017
2018     $contact = get_object_vars($contactObj);
2019
2020     foreach ($params as $key => $value) {
2021       if ($key == 'id' || $key == 'contact_type') {
2022         continue;
2023       }
2024
2025       if (array_key_exists($key, $locationFields)) {
2026         continue;
2027       }
2028       if (in_array($key, [
2029         'email_greeting',
2030         'postal_greeting',
2031         'addressee',
2032       ])) {
2033         // CRM-4575, need to null custom
2034         if ($params["{$key}_id"] != 4) {
2035           $params["{$key}_custom"] = 'null';
2036         }
2037         unset($params[$key]);
2038       }
2039       else {
2040         if ($customFieldId = CRM_Core_BAO_CustomField::getKeyID($key)) {
2041           $custom_params = ['id' => $contact['id'], 'return' => $key];
2042           $getValue = civicrm_api3('Contact', 'getvalue', $custom_params);
2043           if (empty($getValue)) {
2044             unset($getValue);
2045           }
2046         }
2047         else {
2048           $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $key);
2049         }
2050         if ($key == 'contact_source') {
2051           $params['source'] = $params[$key];
2052           unset($params[$key]);
2053         }
2054
2055         if ($modeFill && isset($getValue)) {
2056           unset($params[$key]);
2057           if ($customFieldId) {
2058             // Extra values must be unset to ensure the values are not
2059             // imported.
2060             unset($params['custom'][$customFieldId]);
2061           }
2062         }
2063       }
2064     }
2065
2066     foreach ($locationFields as $locKeys) {
2067       if (isset($params[$locKeys]) && is_array($params[$locKeys])) {
2068         foreach ($params[$locKeys] as $key => $value) {
2069           if ($modeFill) {
2070             $getValue = CRM_Utils_Array::retrieveValueRecursive($contact, $locKeys);
2071
2072             if (isset($getValue)) {
2073               foreach ($getValue as $cnt => $values) {
2074                 if ($locKeys == 'website') {
2075                   if (($getValue[$cnt]['website_type_id'] == $params[$locKeys][$key]['website_type_id'])) {
2076                     unset($params[$locKeys][$key]);
2077                   }
2078                 }
2079                 else {
2080                   if ((!empty($getValue[$cnt]['location_type_id']) && !empty($params[$locKeys][$key]['location_type_id'])) && $getValue[$cnt]['location_type_id'] == $params[$locKeys][$key]['location_type_id']) {
2081                     unset($params[$locKeys][$key]);
2082                   }
2083                 }
2084               }
2085             }
2086           }
2087         }
2088         if (count($params[$locKeys]) == 0) {
2089           unset($params[$locKeys]);
2090         }
2091       }
2092     }
2093   }
2094
2095   /**
2096    * Convert any given date string to default date array.
2097    *
2098    * @param array $params
2099    *   Has given date-format.
2100    * @param array $formatted
2101    *   Store formatted date in this array.
2102    * @param int $dateType
2103    *   Type of date.
2104    * @param string $dateParam
2105    *   Index of params.
2106    */
2107   public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2108     //fix for CRM-2687
2109     CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam);
2110     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
2111   }
2112
2113   /**
2114    * Generate status and error message for unparsed street address records.
2115    *
2116    * @param array $values
2117    *   The array of values belonging to each row.
2118    * @param array $statusFieldName
2119    *   Store formatted date in this array.
2120    * @param $returnCode
2121    *
2122    * @return int
2123    */
2124   public function processMessage(&$values, $statusFieldName, $returnCode) {
2125     if (empty($this->_unparsedStreetAddressContacts)) {
2126       $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '');
2127     }
2128     else {
2129       $errorMessage = ts("Record imported successfully but unable to parse the street address: ");
2130       foreach ($this->_unparsedStreetAddressContacts as $contactInfo => $contactValue) {
2131         $contactUrl = CRM_Utils_System::url('civicrm/contact/add', 'reset=1&action=update&cid=' . $contactValue['id'], TRUE, NULL, FALSE);
2132         $errorMessage .= "\n Contact ID:" . $contactValue['id'] . " <a href=\"$contactUrl\"> " . $contactValue['streetAddress'] . "</a>";
2133       }
2134       array_unshift($values, $errorMessage);
2135       $returnCode = CRM_Import_Parser::UNPARSED_ADDRESS_WARNING;
2136       $this->setImportStatus((int) ($values[count($values) - 1]), 'ERROR', $errorMessage);
2137     }
2138     return $returnCode;
2139   }
2140
2141   /**
2142    * @param $relKey
2143    * @param array $params
2144    *
2145    * @return bool
2146    */
2147   public function checkRelatedContactFields($relKey, $params) {
2148     //avoid blank contact creation.
2149     $allowToCreate = FALSE;
2150
2151     //build the mapper field array.
2152     static $relatedContactFields = [];
2153     if (!isset($relatedContactFields[$relKey])) {
2154       foreach ($this->_mapperRelated as $key => $name) {
2155         if (!$name) {
2156           continue;
2157         }
2158
2159         if (!empty($relatedContactFields[$name]) && !is_array($relatedContactFields[$name])) {
2160           $relatedContactFields[$name] = [];
2161         }
2162         $fldName = $this->_mapperRelatedContactDetails[$key] ?? NULL;
2163         if ($fldName == 'url') {
2164           $fldName = 'website';
2165         }
2166         if ($fldName) {
2167           $relatedContactFields[$name][] = $fldName;
2168         }
2169       }
2170     }
2171
2172     //validate for passed data.
2173     if (is_array($relatedContactFields[$relKey])) {
2174       foreach ($relatedContactFields[$relKey] as $fld) {
2175         if (!empty($params[$fld])) {
2176           $allowToCreate = TRUE;
2177           break;
2178         }
2179       }
2180     }
2181
2182     return $allowToCreate;
2183   }
2184
2185   /**
2186    * get subtypes given the contact type
2187    *
2188    * @param string $contactType
2189    * @return array $subTypes
2190    */
2191   public static function getSubtypes($contactType) {
2192     $subTypes = [];
2193     $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType);
2194
2195     if (count($types) > 0) {
2196       foreach ($types as $type) {
2197         $subTypes[] = $type['name'];
2198       }
2199     }
2200     return $subTypes;
2201   }
2202
2203   /**
2204    * Get the possible contact matches.
2205    *
2206    * 1) the chosen dedupe rule falling back to
2207    * 2) a check for the external ID.
2208    *
2209    * @see https://issues.civicrm.org/jira/browse/CRM-17275
2210    *
2211    * @param array $params
2212    *
2213    * @return array
2214    *   IDs of possible matches.
2215    *
2216    * @throws \CRM_Core_Exception
2217    * @throws \CiviCRM_API3_Exception
2218    */
2219   protected function getPossibleContactMatches($params) {
2220     $extIDMatch = NULL;
2221
2222     if (!empty($params['external_identifier'])) {
2223       // Check for any match on external id, deleted or otherwise.
2224       $extIDContact = civicrm_api3('Contact', 'get', [
2225         'external_identifier' => $params['external_identifier'],
2226         'showAll' => 'all',
2227         'return' => ['id', 'contact_is_deleted'],
2228       ]);
2229       if (isset($extIDContact['id'])) {
2230         $extIDMatch = $extIDContact['id'];
2231
2232         if ($extIDContact['values'][$extIDMatch]['contact_is_deleted'] == 1) {
2233           // If the contact is deleted, update external identifier to be blank
2234           // to avoid key error from MySQL.
2235           $params = ['id' => $extIDMatch, 'external_identifier' => ''];
2236           civicrm_api3('Contact', 'create', $params);
2237
2238           // And now it is no longer a match.
2239           $extIDMatch = NULL;
2240         }
2241       }
2242     }
2243     $checkParams = ['check_permissions' => FALSE, 'match' => $params];
2244     $checkParams['match']['contact_type'] = $this->_contactType;
2245
2246     $possibleMatches = civicrm_api3('Contact', 'duplicatecheck', $checkParams);
2247     if (!$extIDMatch) {
2248       return array_keys($possibleMatches['values']);
2249     }
2250     if ($possibleMatches['count']) {
2251       if (array_key_exists($extIDMatch, $possibleMatches['values'])) {
2252         return [$extIDMatch];
2253       }
2254       throw new CRM_Core_Exception(ts(
2255         'Matching this contact based on the de-dupe rule would cause an external ID conflict'));
2256     }
2257     return [$extIDMatch];
2258   }
2259
2260   /**
2261    * Format the form mapping parameters ready for the parser.
2262    *
2263    * @param int $count
2264    *   Number of rows.
2265    *
2266    * @return array $parserParameters
2267    */
2268   public static function getParameterForParser($count) {
2269     $baseArray = [];
2270     for ($i = 0; $i < $count; $i++) {
2271       $baseArray[$i] = NULL;
2272     }
2273     $parserParameters['mapperLocType'] = $baseArray;
2274     $parserParameters['mapperPhoneType'] = $baseArray;
2275     $parserParameters['mapperImProvider'] = $baseArray;
2276     $parserParameters['mapperWebsiteType'] = $baseArray;
2277     $parserParameters['mapperRelated'] = $baseArray;
2278     $parserParameters['relatedContactType'] = $baseArray;
2279     $parserParameters['relatedContactDetails'] = $baseArray;
2280     $parserParameters['relatedContactLocType'] = $baseArray;
2281     $parserParameters['relatedContactPhoneType'] = $baseArray;
2282     $parserParameters['relatedContactImProvider'] = $baseArray;
2283     $parserParameters['relatedContactWebsiteType'] = $baseArray;
2284
2285     return $parserParameters;
2286
2287   }
2288
2289   /**
2290    * Set field metadata.
2291    */
2292   protected function setFieldMetadata() {
2293     $this->setImportableFieldsMetadata($this->getContactImportMetadata());
2294     // Probably no longer needed but here for now.
2295     $this->_relationships = $this->getRelationships();
2296   }
2297
2298   /**
2299    * @param array $newContact
2300    * @param $statusFieldName
2301    * @param array $values
2302    * @param int $onDuplicate
2303    * @param array $formatted
2304    * @param array $contactFields
2305    *
2306    * @return int
2307    *
2308    * @throws \CRM_Core_Exception
2309    * @throws \CiviCRM_API3_Exception
2310    * @throws \Civi\API\Exception\UnauthorizedException
2311    */
2312   protected function handleDuplicateError(array $newContact, $statusFieldName, array $values, int $onDuplicate, array $formatted, array $contactFields): int {
2313     $urls = [];
2314     // need to fix at some stage and decide if the error will return an
2315     // array or string, crude hack for now
2316     if (is_array($newContact['error_message']['params'][0])) {
2317       $cids = $newContact['error_message']['params'][0];
2318     }
2319     else {
2320       $cids = explode(',', $newContact['error_message']['params'][0]);
2321     }
2322
2323     foreach ($cids as $cid) {
2324       $urls[] = CRM_Utils_System::url('civicrm/contact/view', 'reset=1&cid=' . $cid, TRUE);
2325     }
2326
2327     $url_string = implode("\n", $urls);
2328
2329     // If we duplicate more than one record, skip no matter what
2330     if (count($cids) > 1) {
2331       $errorMessage = ts('Record duplicates multiple contacts');
2332       //combine error msg to avoid mismatch between error file columns.
2333       $errorMessage .= "\n" . $url_string;
2334       array_unshift($values, $errorMessage);
2335       $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2336       return CRM_Import_Parser::ERROR;
2337     }
2338
2339     // Params only had one id, so shift it out
2340     $contactId = array_shift($cids);
2341     $cid = NULL;
2342
2343     $vals = ['contact_id' => $contactId];
2344
2345     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_REPLACE) {
2346       civicrm_api('contact', 'delete', $vals);
2347       $cid = CRM_Contact_BAO_Contact::createProfileContact($formatted, $contactFields, $contactId, NULL, NULL, $formatted['contact_type']);
2348     }
2349     if (in_array((int) $onDuplicate, [CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::DUPLICATE_FILL], TRUE)) {
2350       $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $contactId);
2351     }
2352     // else skip does nothing and just returns an error code.
2353     if ($cid) {
2354       $contact = [
2355         'contact_id' => $cid,
2356       ];
2357       $defaults = [];
2358       $newContact = CRM_Contact_BAO_Contact::retrieve($contact, $defaults);
2359     }
2360
2361     if (civicrm_error($newContact)) {
2362       if (empty($newContact['error_message']['params'])) {
2363         // different kind of error other than DUPLICATE
2364         $errorMessage = $newContact['error_message'];
2365         array_unshift($values, $errorMessage);
2366         $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $errorMessage);
2367         return CRM_Import_Parser::ERROR;
2368       }
2369
2370       $contactID = $newContact['error_message']['params'][0];
2371       if (is_array($contactID)) {
2372         $contactID = array_pop($contactID);
2373       }
2374       if (!in_array($contactID, $this->_newContacts)) {
2375         $this->_newContacts[] = $contactID;
2376       }
2377     }
2378     //CRM-262 No Duplicate Checking
2379     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) {
2380       array_unshift($values, $url_string);
2381       $this->setImportStatus((int) $values[count($values) - 1], 'DUPLICATE', 'Skipping duplicate record');
2382       return CRM_Import_Parser::DUPLICATE;
2383     }
2384
2385     $this->setImportStatus((int) $values[count($values) - 1], 'Imported', '');
2386     //return warning if street address is not parsed, CRM-5886
2387     return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID);
2388   }
2389
2390   /**
2391    * Validate a formatted contact parameter list.
2392    *
2393    * @param array $params
2394    *   Structured parameter list (as in crm_format_params).
2395    *
2396    * @throw CRM_Core_Error
2397    */
2398   public function deprecated_validate_formatted_contact(&$params): void {
2399     // Look for offending email addresses
2400
2401     if (array_key_exists('email', $params)) {
2402       foreach ($params['email'] as $count => $values) {
2403         if (!is_array($values)) {
2404           continue;
2405         }
2406         if ($email = CRM_Utils_Array::value('email', $values)) {
2407           // validate each email
2408           if (!CRM_Utils_Rule::email($email)) {
2409             throw new CRM_Core_Exception('No valid email address');
2410           }
2411
2412           // check for loc type id.
2413           if (empty($values['location_type_id'])) {
2414             throw new CRM_Core_Exception('Location Type Id missing.');
2415           }
2416         }
2417       }
2418     }
2419
2420     // Validate custom data fields
2421     if (array_key_exists('custom', $params) && is_array($params['custom'])) {
2422       foreach ($params['custom'] as $key => $custom) {
2423         if (is_array($custom)) {
2424           foreach ($custom as $fieldId => $value) {
2425             $valid = CRM_Core_BAO_CustomValue::typecheck(CRM_Utils_Array::value('type', $value),
2426               CRM_Utils_Array::value('value', $value)
2427             );
2428             if (!$valid && $value['is_required']) {
2429               throw new CRM_Core_Exception('Invalid value for custom field \'' .
2430                 $custom['name'] . '\''
2431               );
2432             }
2433             if (CRM_Utils_Array::value('type', $custom) == 'Date') {
2434               $params['custom'][$key][$fieldId]['value'] = str_replace('-', '', $params['custom'][$key][$fieldId]['value']);
2435             }
2436           }
2437         }
2438       }
2439     }
2440   }
2441
2442   /**
2443    * @param array $params
2444    * @param bool $dupeCheck
2445    * @param null|int $dedupeRuleGroupID
2446    *
2447    * @throws \CRM_Core_Exception
2448    */
2449   public function deprecated_contact_check_params(
2450     &$params,
2451     $dupeCheck = TRUE,
2452     $dedupeRuleGroupID = NULL) {
2453
2454     $requiredCheck = TRUE;
2455
2456     if (isset($params['id']) && is_numeric($params['id'])) {
2457       $requiredCheck = FALSE;
2458     }
2459     if ($requiredCheck) {
2460       $required = [
2461         'Individual' => [
2462           ['first_name', 'last_name'],
2463           'email',
2464         ],
2465         'Household' => [
2466           'household_name',
2467         ],
2468         'Organization' => [
2469           'organization_name',
2470         ],
2471       ];
2472
2473       // contact_type has a limited number of valid values
2474       if (empty($params['contact_type'])) {
2475         throw new CRM_Core_Exception("No Contact Type");
2476       }
2477       $fields = $required[$params['contact_type']] ?? NULL;
2478       if ($fields == NULL) {
2479         throw new CRM_Core_Exception("Invalid Contact Type: {$params['contact_type']}");
2480       }
2481
2482       if ($csType = CRM_Utils_Array::value('contact_sub_type', $params)) {
2483         if (!(CRM_Contact_BAO_ContactType::isExtendsContactType($csType, $params['contact_type']))) {
2484           throw new CRM_Core_Exception("Invalid or Mismatched Contact Subtype: " . implode(', ', (array) $csType));
2485         }
2486       }
2487
2488       if (empty($params['contact_id']) && !empty($params['id'])) {
2489         $valid = FALSE;
2490         $error = '';
2491         foreach ($fields as $field) {
2492           if (is_array($field)) {
2493             $valid = TRUE;
2494             foreach ($field as $element) {
2495               if (empty($params[$element])) {
2496                 $valid = FALSE;
2497                 $error .= $element;
2498                 break;
2499               }
2500             }
2501           }
2502           else {
2503             if (!empty($params[$field])) {
2504               $valid = TRUE;
2505             }
2506           }
2507           if ($valid) {
2508             break;
2509           }
2510         }
2511
2512         if (!$valid) {
2513           throw new CRM_Core_Exception("Required fields not found for {$params['contact_type']} : $error");
2514         }
2515       }
2516     }
2517
2518     if ($dupeCheck) {
2519       // @todo switch to using api version
2520       // $dupes = civicrm_api3('Contact', 'duplicatecheck', (array('match' => $params, 'dedupe_rule_id' => $dedupeRuleGroupID)));
2521       // $ids = $dupes['count'] ? implode(',', array_keys($dupes['values'])) : NULL;
2522       $ids = CRM_Contact_BAO_Contact::getDuplicateContacts($params, $params['contact_type'], 'Unsupervised', [], CRM_Utils_Array::value('check_permissions', $params), $dedupeRuleGroupID);
2523       if ($ids != NULL) {
2524         $error = CRM_Core_Error::createError("Found matching contacts: " . implode(',', $ids),
2525           CRM_Core_Error::DUPLICATE_CONTACT,
2526           'Fatal', $ids
2527         );
2528         return civicrm_api3_create_error($error->pop());
2529       }
2530     }
2531
2532     // check for organisations with same name
2533     if (!empty($params['current_employer'])) {
2534       $organizationParams = ['organization_name' => $params['current_employer']];
2535       $dupeIds = CRM_Contact_BAO_Contact::getDuplicateContacts($organizationParams, 'Organization', 'Supervised', [], FALSE);
2536
2537       // check for mismatch employer name and id
2538       if (!empty($params['employer_id']) && !in_array($params['employer_id'], $dupeIds)
2539       ) {
2540         throw new CRM_Core_Exception('Employer name and Employer id Mismatch');
2541       }
2542
2543       // show error if multiple organisation with same name exist
2544       if (empty($params['employer_id']) && (count($dupeIds) > 1)
2545       ) {
2546         return civicrm_api3_create_error('Found more than one Organisation with same Name.');
2547       }
2548     }
2549   }
2550
2551   /**
2552    * Run import.
2553    *
2554    * @param string $tableName
2555    * @param array $mapper
2556    * @param int $mode
2557    * @param int $contactType
2558    * @param string $primaryKeyName
2559    * @param string $statusFieldName
2560    * @param int $onDuplicate
2561    * @param int $statusID
2562    * @param int $totalRowCount
2563    * @param bool $doGeocodeAddress
2564    * @param int $timeout
2565    * @param string $contactSubType
2566    * @param int $dedupeRuleGroupID
2567    *
2568    * @return mixed
2569    */
2570   public function run(
2571     $tableName,
2572     $mapper = [],
2573     $mode = self::MODE_PREVIEW,
2574     $contactType = self::CONTACT_INDIVIDUAL,
2575     $primaryKeyName = '_id',
2576     $statusFieldName = '_status',
2577     $onDuplicate = self::DUPLICATE_SKIP,
2578     $statusID = NULL,
2579     $totalRowCount = NULL,
2580     $doGeocodeAddress = FALSE,
2581     $timeout = CRM_Contact_Import_Parser_Contact::DEFAULT_TIMEOUT,
2582     $contactSubType = NULL,
2583     $dedupeRuleGroupID = NULL
2584   ) {
2585
2586     // TODO: Make the timeout actually work
2587     $this->_onDuplicate = $onDuplicate;
2588     $this->_dedupeRuleGroupID = $dedupeRuleGroupID;
2589     // Since $this->_contactType is still being called directly do a get call
2590     // here to make sure it is instantiated.
2591     $this->getContactType();
2592     $this->getContactSubType();
2593
2594     $this->init();
2595
2596     $this->_rowCount = 0;
2597     $this->_invalidRowCount = $this->_validCount = 0;
2598     $this->_totalCount = $this->_conflictCount = 0;
2599
2600     $this->_errors = [];
2601     $this->_warnings = [];
2602     $this->_conflicts = [];
2603     $this->_unparsedAddresses = [];
2604
2605     // Transitional support for deprecating table_name (and other fields)
2606     // form input - the goal is to load them from userJob - but eventually
2607     // we will just load the datasource object and this code will not know the
2608     // table name.
2609     if (!$tableName && $this->userJobID) {
2610       $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
2611     }
2612
2613     $this->_tableName = $tableName;
2614     $this->_primaryKeyName = '_id';
2615     $this->_statusFieldName = '_status';
2616
2617     if ($mode == self::MODE_MAPFIELD) {
2618       $this->_rows = [];
2619     }
2620     else {
2621       $this->_activeFieldCount = count($this->_activeFields);
2622     }
2623
2624     if ($statusID) {
2625       $this->progressImport($statusID);
2626       $startTimestamp = $currTimestamp = $prevTimestamp = time();
2627     }
2628     // get the contents of the temp. import table
2629     $query = "SELECT * FROM $tableName";
2630     if ($mode == self::MODE_IMPORT) {
2631       $query .= " WHERE $statusFieldName = 'NEW'";
2632     }
2633     if ($this->_maxLinesToProcess > 0) {
2634       // Note this would only be the case in MapForm mode, where it is set to 100
2635       // rows. In fact mapField really only needs 2 rows - the reason for
2636       // 100 seems to be that the other import classes are processing a
2637       // csv file, and there was a concern that some rows might have more
2638       // columns than others - hence checking 100 rows perhaps seemed like
2639       // a good precaution presumably when determining the activeFieldsCount
2640       // which is the number of columns a row might have.
2641       // However, the mapField class may no longer use activeFieldsCount for contact
2642       // to be continued....
2643       $query .= ' LIMIT ' . $this->_maxLinesToProcess;
2644     }
2645
2646     $result = CRM_Core_DAO::executeQuery($query);
2647
2648     while ($result->fetch()) {
2649       $values = array_values($result->toArray());
2650       $this->_rowCount++;
2651
2652       /* trim whitespace around the values */
2653       foreach ($values as $k => $v) {
2654         $values[$k] = trim($v, " \t\r\n");
2655       }
2656
2657       $this->_totalCount++;
2658
2659       if ($mode == self::MODE_MAPFIELD) {
2660         $returnCode = CRM_Import_Parser::VALID;
2661       }
2662       elseif ($mode == self::MODE_PREVIEW) {
2663         $returnCode = $this->preview($values);
2664       }
2665       elseif ($mode == self::MODE_SUMMARY) {
2666         $returnCode = $this->summary($values);
2667       }
2668       elseif ($mode == self::MODE_IMPORT) {
2669         try {
2670           $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
2671         }
2672         catch (CiviCRM_API3_Exception $e) {
2673           // When we catch errors here we are not adding to the errors array - mostly
2674           // because that will become obsolete once https://github.com/civicrm/civicrm-core/pull/23292
2675           // is merged and this will replace it as the main way to handle errors (ie. update the table
2676           // and move on).
2677           $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $e->getMessage());
2678         }
2679         if ($statusID && (($this->_rowCount % 50) == 0)) {
2680           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
2681         }
2682       }
2683       else {
2684         $returnCode = self::ERROR;
2685       }
2686
2687       // note that a line could be valid but still produce a warning
2688       if ($returnCode & self::VALID) {
2689         $this->_validCount++;
2690         if ($mode == self::MODE_MAPFIELD) {
2691           $this->_rows[] = $values;
2692           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
2693         }
2694       }
2695
2696       if ($returnCode & self::ERROR) {
2697         $this->_invalidRowCount++;
2698         array_unshift($values, $this->_rowCount);
2699         $this->_errors[] = $values;
2700       }
2701
2702       if ($returnCode & self::CONFLICT) {
2703         $this->_conflictCount++;
2704         array_unshift($values, $this->_rowCount);
2705         $this->_conflicts[] = $values;
2706       }
2707
2708       if ($returnCode & self::NO_MATCH) {
2709         $this->_unMatchCount++;
2710         array_unshift($values, $this->_rowCount);
2711         $this->_unMatch[] = $values;
2712       }
2713
2714       if ($returnCode & self::DUPLICATE) {
2715         $this->_duplicateCount++;
2716         array_unshift($values, $this->_rowCount);
2717         $this->_duplicates[] = $values;
2718         if ($onDuplicate != self::DUPLICATE_SKIP) {
2719           $this->_validCount++;
2720         }
2721       }
2722
2723       if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
2724         $this->_unparsedAddressCount++;
2725         array_unshift($values, $this->_rowCount);
2726         $this->_unparsedAddresses[] = $values;
2727       }
2728       // we give the derived class a way of aborting the process
2729       // note that the return code could be multiple code or'ed together
2730       if ($returnCode & self::STOP) {
2731         break;
2732       }
2733
2734       // see if we've hit our timeout yet
2735       /* if ( $the_thing_with_the_stuff ) {
2736       do_something( );
2737       } */
2738     }
2739
2740     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
2741       $customHeaders = $mapper;
2742
2743       $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType);
2744       foreach ($customHeaders as $key => $value) {
2745         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
2746           $customHeaders[$key] = $customfields[$id][0];
2747         }
2748       }
2749
2750       if ($this->_invalidRowCount) {
2751         // removed view url for invlaid contacts
2752         $headers = array_merge([
2753           ts('Line Number'),
2754           ts('Reason'),
2755         ], $customHeaders);
2756         $this->_errorFileName = self::errorFileName(self::ERROR);
2757         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
2758       }
2759       if ($this->_conflictCount) {
2760         $headers = array_merge([
2761           ts('Line Number'),
2762           ts('Reason'),
2763         ], $customHeaders);
2764         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
2765         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
2766       }
2767       if ($this->_duplicateCount) {
2768         $headers = array_merge([
2769           ts('Line Number'),
2770           ts('View Contact URL'),
2771         ], $customHeaders);
2772
2773         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
2774         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
2775       }
2776       if ($this->_unMatchCount) {
2777         $headers = array_merge([
2778           ts('Line Number'),
2779           ts('Reason'),
2780         ], $customHeaders);
2781
2782         $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH);
2783         self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch);
2784       }
2785       if ($this->_unparsedAddressCount) {
2786         $headers = array_merge([
2787           ts('Line Number'),
2788           ts('Contact Edit URL'),
2789         ], $customHeaders);
2790         $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING);
2791         self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses);
2792       }
2793     }
2794   }
2795
2796   /**
2797    * Given a list of the importable field keys that the user has selected.
2798    * set the active fields array to this list
2799    *
2800    * @param array $fieldKeys
2801    *   Mapped array of values.
2802    */
2803   public function setActiveFields($fieldKeys) {
2804     $this->_activeFieldCount = count($fieldKeys);
2805     foreach ($fieldKeys as $key) {
2806       if (empty($this->_fields[$key])) {
2807         $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
2808       }
2809       else {
2810         $this->_activeFields[] = clone($this->_fields[$key]);
2811       }
2812     }
2813   }
2814
2815   /**
2816    * @param $elements
2817    */
2818   public function setActiveFieldLocationTypes($elements) {
2819     for ($i = 0; $i < count($elements); $i++) {
2820       $this->_activeFields[$i]->_hasLocationType = $elements[$i];
2821     }
2822   }
2823
2824   /**
2825    * @param $elements
2826    */
2827
2828   /**
2829    * @param $elements
2830    */
2831   public function setActiveFieldPhoneTypes($elements) {
2832     for ($i = 0; $i < count($elements); $i++) {
2833       $this->_activeFields[$i]->_phoneType = $elements[$i];
2834     }
2835   }
2836
2837   /**
2838    * @param $elements
2839    */
2840   public function setActiveFieldWebsiteTypes($elements) {
2841     for ($i = 0; $i < count($elements); $i++) {
2842       $this->_activeFields[$i]->_websiteType = $elements[$i];
2843     }
2844   }
2845
2846   /**
2847    * Set IM Service Provider type fields.
2848    *
2849    * @param array $elements
2850    *   IM service provider type ids.
2851    */
2852   public function setActiveFieldImProviders($elements) {
2853     for ($i = 0; $i < count($elements); $i++) {
2854       $this->_activeFields[$i]->_imProvider = $elements[$i];
2855     }
2856   }
2857
2858   /**
2859    * @param $elements
2860    */
2861   public function setActiveFieldRelated($elements) {
2862     for ($i = 0; $i < count($elements); $i++) {
2863       $this->_activeFields[$i]->_related = $elements[$i];
2864     }
2865   }
2866
2867   /**
2868    * @param $elements
2869    */
2870   public function setActiveFieldRelatedContactType($elements) {
2871     for ($i = 0; $i < count($elements); $i++) {
2872       $this->_activeFields[$i]->_relatedContactType = $elements[$i];
2873     }
2874   }
2875
2876   /**
2877    * @param $elements
2878    */
2879   public function setActiveFieldRelatedContactDetails($elements) {
2880     for ($i = 0; $i < count($elements); $i++) {
2881       $this->_activeFields[$i]->_relatedContactDetails = $elements[$i];
2882     }
2883   }
2884
2885   /**
2886    * @param $elements
2887    */
2888   public function setActiveFieldRelatedContactLocType($elements) {
2889     for ($i = 0; $i < count($elements); $i++) {
2890       $this->_activeFields[$i]->_relatedContactLocType = $elements[$i];
2891     }
2892   }
2893
2894   /**
2895    * Set active field for related contact's phone type.
2896    *
2897    * @param array $elements
2898    */
2899   public function setActiveFieldRelatedContactPhoneType($elements) {
2900     for ($i = 0; $i < count($elements); $i++) {
2901       $this->_activeFields[$i]->_relatedContactPhoneType = $elements[$i];
2902     }
2903   }
2904
2905   /**
2906    * @param $elements
2907    */
2908   public function setActiveFieldRelatedContactWebsiteType($elements) {
2909     for ($i = 0; $i < count($elements); $i++) {
2910       $this->_activeFields[$i]->_relatedContactWebsiteType = $elements[$i];
2911     }
2912   }
2913
2914   /**
2915    * Set IM Service Provider type fields for related contacts.
2916    *
2917    * @param array $elements
2918    *   IM service provider type ids of related contact.
2919    */
2920   public function setActiveFieldRelatedContactImProvider($elements) {
2921     for ($i = 0; $i < count($elements); $i++) {
2922       $this->_activeFields[$i]->_relatedContactImProvider = $elements[$i];
2923     }
2924   }
2925
2926   /**
2927    * Format the field values for input to the api.
2928    *
2929    * @return array
2930    *   (reference ) associative array of name/value pairs
2931    */
2932   public function &getActiveFieldParams() {
2933     $params = [];
2934
2935     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
2936       if ($this->_activeFields[$i]->_name == 'do_not_import') {
2937         continue;
2938       }
2939
2940       if (isset($this->_activeFields[$i]->_value)) {
2941         if (isset($this->_activeFields[$i]->_hasLocationType)) {
2942           if (!isset($params[$this->_activeFields[$i]->_name])) {
2943             $params[$this->_activeFields[$i]->_name] = [];
2944           }
2945
2946           $value = [
2947             $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
2948             'location_type_id' => $this->_activeFields[$i]->_hasLocationType,
2949           ];
2950
2951           if (isset($this->_activeFields[$i]->_phoneType)) {
2952             $value['phone_type_id'] = $this->_activeFields[$i]->_phoneType;
2953           }
2954
2955           // get IM service Provider type id
2956           if (isset($this->_activeFields[$i]->_imProvider)) {
2957             $value['provider_id'] = $this->_activeFields[$i]->_imProvider;
2958           }
2959
2960           $params[$this->_activeFields[$i]->_name][] = $value;
2961         }
2962         elseif (isset($this->_activeFields[$i]->_websiteType)) {
2963           $value = [
2964             $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
2965             'website_type_id' => $this->_activeFields[$i]->_websiteType,
2966           ];
2967
2968           $params[$this->_activeFields[$i]->_name][] = $value;
2969         }
2970
2971         if (!isset($params[$this->_activeFields[$i]->_name])) {
2972           if (!isset($this->_activeFields[$i]->_related)) {
2973             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
2974           }
2975         }
2976
2977         //minor fix for CRM-4062
2978         if (isset($this->_activeFields[$i]->_related)) {
2979           if (!isset($params[$this->_activeFields[$i]->_related])) {
2980             $params[$this->_activeFields[$i]->_related] = [];
2981           }
2982
2983           if (!isset($params[$this->_activeFields[$i]->_related]['contact_type']) && !empty($this->_activeFields[$i]->_relatedContactType)) {
2984             $params[$this->_activeFields[$i]->_related]['contact_type'] = $this->_activeFields[$i]->_relatedContactType;
2985           }
2986
2987           if (isset($this->_activeFields[$i]->_relatedContactLocType) && !empty($this->_activeFields[$i]->_value)) {
2988             if (!empty($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails]) &&
2989               !is_array($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])
2990             ) {
2991               $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = [];
2992             }
2993             $value = [
2994               $this->_activeFields[$i]->_relatedContactDetails => $this->_activeFields[$i]->_value,
2995               'location_type_id' => $this->_activeFields[$i]->_relatedContactLocType,
2996             ];
2997
2998             if (isset($this->_activeFields[$i]->_relatedContactPhoneType)) {
2999               $value['phone_type_id'] = $this->_activeFields[$i]->_relatedContactPhoneType;
3000             }
3001
3002             // get IM service Provider type id for related contact
3003             if (isset($this->_activeFields[$i]->_relatedContactImProvider)) {
3004               $value['provider_id'] = $this->_activeFields[$i]->_relatedContactImProvider;
3005             }
3006
3007             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = $value;
3008           }
3009           elseif (isset($this->_activeFields[$i]->_relatedContactWebsiteType)) {
3010             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = [
3011               'url' => $this->_activeFields[$i]->_value,
3012               'website_type_id' => $this->_activeFields[$i]->_relatedContactWebsiteType,
3013             ];
3014           }
3015           elseif (empty($this->_activeFields[$i]->_value) && isset($this->_activeFields[$i]->_relatedContactLocType)) {
3016             if (empty($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])) {
3017               $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = [];
3018             }
3019           }
3020           else {
3021             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = $this->_activeFields[$i]->_value;
3022           }
3023         }
3024       }
3025     }
3026
3027     return $params;
3028   }
3029
3030   /**
3031    * @param string $name
3032    * @param $title
3033    * @param int $type
3034    * @param string $headerPattern
3035    * @param string $dataPattern
3036    * @param bool $hasLocationType
3037    */
3038   public function addField(
3039     $name, $title, $type = CRM_Utils_Type::T_INT,
3040     $headerPattern = '//', $dataPattern = '//',
3041     $hasLocationType = FALSE
3042   ) {
3043     $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
3044     if (empty($name)) {
3045       $this->_fields['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
3046     }
3047   }
3048
3049   /**
3050    * Store parser values.
3051    *
3052    * @param CRM_Core_Session $store
3053    *
3054    * @param int $mode
3055    */
3056   public function set($store, $mode = self::MODE_SUMMARY) {
3057     // @todo - this params are being set here because they were / possibly still
3058     // are in some places being accessed by forms later in the flow
3059     // ie CRM_Contact_Import_Form_MapField, CRM_Contact_Import_Form_Preview
3060     // or CRM_Contact_Import_Form_Summary using `$this->get()
3061     // which was the old way of saving values submitted on this form such that
3062     // the other forms could access them. Now they should use
3063     // `getSubmittedValue` or simply not get them if the only
3064     // reason is to pass to the Parser which can itself
3065     // call 'getSubmittedValue'
3066     // Once the mentioned forms no longer call $this->get() all this 'setting'
3067     // is obsolete.
3068     $store->set('rowCount', $this->_rowCount);
3069     $store->set('fieldTypes', $this->getSelectTypes());
3070
3071     $store->set('columnCount', $this->_activeFieldCount);
3072
3073     $store->set('totalRowCount', $this->_totalCount);
3074     $store->set('validRowCount', $this->_validCount);
3075     $store->set('invalidRowCount', $this->_invalidRowCount);
3076     $store->set('conflictRowCount', $this->_conflictCount);
3077     $store->set('unMatchCount', $this->_unMatchCount);
3078
3079     switch ($this->_contactType) {
3080       case 'Individual':
3081         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
3082         break;
3083
3084       case 'Household':
3085         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
3086         break;
3087
3088       case 'Organization':
3089         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
3090     }
3091
3092     if ($this->_invalidRowCount) {
3093       $store->set('errorsFileName', $this->_errorFileName);
3094     }
3095     if ($this->_conflictCount) {
3096       $store->set('conflictsFileName', $this->_conflictFileName);
3097     }
3098     if (isset($this->_rows) && !empty($this->_rows)) {
3099       $store->set('dataValues', $this->_rows);
3100     }
3101
3102     if ($this->_unMatchCount) {
3103       $store->set('mismatchFileName', $this->_misMatchFilemName);
3104     }
3105
3106     if ($mode == self::MODE_IMPORT) {
3107       $store->set('duplicateRowCount', $this->_duplicateCount);
3108       $store->set('unparsedAddressCount', $this->_unparsedAddressCount);
3109       if ($this->_duplicateCount) {
3110         $store->set('duplicatesFileName', $this->_duplicateFileName);
3111       }
3112       if ($this->_unparsedAddressCount) {
3113         $store->set('errorsFileName', $this->_errorFileName);
3114       }
3115     }
3116     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
3117   }
3118
3119   /**
3120    * Export data to a CSV file.
3121    *
3122    * @param string $fileName
3123    * @param array $header
3124    * @param array $data
3125    */
3126   public static function exportCSV($fileName, $header, $data) {
3127
3128     if (file_exists($fileName) && !is_writable($fileName)) {
3129       CRM_Core_Error::movedSiteError($fileName);
3130     }
3131     //hack to remove '_status', '_statusMsg' and '_id' from error file
3132     $errorValues = [];
3133     $dbRecordStatus = ['IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW'];
3134     foreach ($data as $rowCount => $rowValues) {
3135       $count = 0;
3136       foreach ($rowValues as $key => $val) {
3137         if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
3138           break;
3139         }
3140         $errorValues[$rowCount][$key] = $val;
3141         $count++;
3142       }
3143     }
3144     $data = $errorValues;
3145
3146     $output = [];
3147     $fd = fopen($fileName, 'w');
3148
3149     foreach ($header as $key => $value) {
3150       $header[$key] = "\"$value\"";
3151     }
3152     $config = CRM_Core_Config::singleton();
3153     $output[] = implode($config->fieldSeparator, $header);
3154
3155     foreach ($data as $datum) {
3156       foreach ($datum as $key => $value) {
3157         $datum[$key] = "\"$value\"";
3158       }
3159       $output[] = implode($config->fieldSeparator, $datum);
3160     }
3161     fwrite($fd, implode("\n", $output));
3162     fclose($fd);
3163   }
3164
3165   /**
3166    * Set the import status for the given record.
3167    *
3168    * If this is a sql import then the sql table will be used and the update
3169    * will not happen as the relevant fields don't exist in the table - hence
3170    * the checks that statusField & primary key are set.
3171    *
3172    * @param int $id
3173    * @param string $status
3174    * @param string $message
3175    */
3176   public function setImportStatus(int $id, string $status, string $message): void {
3177     if ($this->_statusFieldName && $this->_primaryKeyName) {
3178       CRM_Core_DAO::executeQuery("
3179         UPDATE $this->_tableName
3180         SET $this->_statusFieldName = %1,
3181           {$this->_statusFieldName}Msg = %2
3182         WHERE  $this->_primaryKeyName = %3
3183       ", [
3184         1 => [$status, 'String'],
3185         2 => [$message, 'String'],
3186         3 => [$id, 'Integer'],
3187       ]);
3188     }
3189   }
3190
3191   /**
3192    * Format contact parameters.
3193    *
3194    * @todo this function needs re-writing & re-merging into the main function.
3195    *
3196    * Here be dragons.
3197    *
3198    * @param array $values
3199    * @param array $params
3200    *
3201    * @return bool
3202    */
3203   protected function formatContactParameters(&$values, &$params) {
3204     // Crawl through the possible classes:
3205     // Contact
3206     //      Individual
3207     //      Household
3208     //      Organization
3209     //          Location
3210     //              Address
3211     //              Email
3212     //              Phone
3213     //              IM
3214     //      Note
3215     //      Custom
3216
3217     // first add core contact values since for other Civi modules they are not added
3218     $contactFields = CRM_Contact_DAO_Contact::fields();
3219     _civicrm_api3_store_values($contactFields, $values, $params);
3220
3221     if (isset($values['contact_type'])) {
3222       // we're an individual/household/org property
3223
3224       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
3225
3226       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
3227       return TRUE;
3228     }
3229
3230     // Cache the various object fields
3231     // @todo - remove this after confirming this is just a compilation of other-wise-cached fields.
3232     static $fields = [];
3233
3234     if (isset($values['individual_prefix'])) {
3235       if (!empty($params['prefix_id'])) {
3236         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
3237         $params['prefix'] = $prefixes[$params['prefix_id']];
3238       }
3239       else {
3240         $params['prefix'] = $values['individual_prefix'];
3241       }
3242       return TRUE;
3243     }
3244
3245     if (isset($values['individual_suffix'])) {
3246       if (!empty($params['suffix_id'])) {
3247         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
3248         $params['suffix'] = $suffixes[$params['suffix_id']];
3249       }
3250       else {
3251         $params['suffix'] = $values['individual_suffix'];
3252       }
3253       return TRUE;
3254     }
3255
3256     // CRM-4575
3257     if (isset($values['email_greeting'])) {
3258       if (!empty($params['email_greeting_id'])) {
3259         $emailGreetingFilter = [
3260           'contact_type' => $params['contact_type'] ?? NULL,
3261           'greeting_type' => 'email_greeting',
3262         ];
3263         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
3264         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
3265       }
3266       else {
3267         $params['email_greeting'] = $values['email_greeting'];
3268       }
3269
3270       return TRUE;
3271     }
3272
3273     if (isset($values['postal_greeting'])) {
3274       if (!empty($params['postal_greeting_id'])) {
3275         $postalGreetingFilter = [
3276           'contact_type' => $params['contact_type'] ?? NULL,
3277           'greeting_type' => 'postal_greeting',
3278         ];
3279         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
3280         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
3281       }
3282       else {
3283         $params['postal_greeting'] = $values['postal_greeting'];
3284       }
3285       return TRUE;
3286     }
3287
3288     if (isset($values['addressee'])) {
3289       $params['addressee'] = $values['addressee'];
3290       return TRUE;
3291     }
3292
3293     if (isset($values['gender'])) {
3294       if (!empty($params['gender_id'])) {
3295         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
3296         $params['gender'] = $genders[$params['gender_id']];
3297       }
3298       else {
3299         $params['gender'] = $values['gender'];
3300       }
3301       return TRUE;
3302     }
3303
3304     if (!empty($values['preferred_communication_method'])) {
3305       $comm = [];
3306       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
3307
3308       $preffComm = explode(',', $values['preferred_communication_method']);
3309       foreach ($preffComm as $v) {
3310         $v = strtolower(trim($v));
3311         if (array_key_exists($v, $pcm)) {
3312           $comm[$pcm[$v]] = 1;
3313         }
3314       }
3315
3316       $params['preferred_communication_method'] = $comm;
3317       return TRUE;
3318     }
3319
3320     // format the website params.
3321     if (!empty($values['url'])) {
3322       static $websiteFields;
3323       if (!is_array($websiteFields)) {
3324         $websiteFields = CRM_Core_DAO_Website::fields();
3325       }
3326       if (!array_key_exists('website', $params) ||
3327         !is_array($params['website'])
3328       ) {
3329         $params['website'] = [];
3330       }
3331
3332       $websiteCount = count($params['website']);
3333       _civicrm_api3_store_values($websiteFields, $values,
3334         $params['website'][++$websiteCount]
3335       );
3336
3337       return TRUE;
3338     }
3339
3340     if (isset($values['note'])) {
3341       // add a note field
3342       if (!isset($params['note'])) {
3343         $params['note'] = [];
3344       }
3345       $noteBlock = count($params['note']) + 1;
3346
3347       $params['note'][$noteBlock] = [];
3348       if (!isset($fields['Note'])) {
3349         $fields['Note'] = CRM_Core_DAO_Note::fields();
3350       }
3351
3352       // get the current logged in civicrm user
3353       $session = CRM_Core_Session::singleton();
3354       $userID = $session->get('userID');
3355
3356       if ($userID) {
3357         $values['contact_id'] = $userID;
3358       }
3359
3360       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
3361
3362       return TRUE;
3363     }
3364
3365     // Check for custom field values
3366     $customFields = CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
3367       FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
3368     );
3369
3370     foreach ($values as $key => $value) {
3371       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
3372         // check if it's a valid custom field id
3373
3374         if (!array_key_exists($customFieldID, $customFields)) {
3375           return civicrm_api3_create_error('Invalid custom field ID');
3376         }
3377         else {
3378           $params[$key] = $value;
3379         }
3380       }
3381     }
3382     return TRUE;
3383   }
3384
3385   /**
3386    * Format location block ready for importing.
3387    *
3388    * There is some test coverage for this in CRM_Contact_Import_Parser_ContactTest
3389    * e.g. testImportPrimaryAddress.
3390    *
3391    * @param array $values
3392    * @param array $params
3393    *
3394    * @return bool
3395    */
3396   protected function formatLocationBlock(&$values, &$params) {
3397     $blockTypes = [
3398       'phone' => 'Phone',
3399       'email' => 'Email',
3400       'im' => 'IM',
3401       'openid' => 'OpenID',
3402       'phone_ext' => 'Phone',
3403     ];
3404     foreach ($blockTypes as $blockFieldName => $block) {
3405       if (!array_key_exists($blockFieldName, $values)) {
3406         continue;
3407       }
3408       $blockIndex = $values['location_type_id'] . (!empty($values['phone_type_id']) ? '_' . $values['phone_type_id'] : '');
3409
3410       // block present in value array.
3411       if (!array_key_exists($blockFieldName, $params) || !is_array($params[$blockFieldName])) {
3412         $params[$blockFieldName] = [];
3413       }
3414
3415       $fields[$block] = $this->getMetadataForEntity($block);
3416
3417       // copy value to dao field name.
3418       if ($blockFieldName == 'im') {
3419         $values['name'] = $values[$blockFieldName];
3420       }
3421
3422       _civicrm_api3_store_values($fields[$block], $values,
3423         $params[$blockFieldName][$blockIndex]
3424       );
3425
3426       $this->fillPrimary($params[$blockFieldName][$blockIndex], $values, $block, CRM_Utils_Array::value('id', $params));
3427
3428       if (empty($params['id']) && (count($params[$blockFieldName]) == 1)) {
3429         $params[$blockFieldName][$blockIndex]['is_primary'] = TRUE;
3430       }
3431
3432       // we only process single block at a time.
3433       return TRUE;
3434     }
3435
3436     // handle address fields.
3437     if (!array_key_exists('address', $params) || !is_array($params['address'])) {
3438       $params['address'] = [];
3439     }
3440
3441     // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
3442     // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
3443     // the address in CRM_Core_BAO_Address::create method
3444     if (!empty($values['location_type_id'])) {
3445       static $customFields = [];
3446       if (empty($customFields)) {
3447         $customFields = CRM_Core_BAO_CustomField::getFields('Address');
3448       }
3449       // make a copy of values, as we going to make changes
3450       $newValues = $values;
3451       foreach ($values as $key => $val) {
3452         $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
3453         if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
3454
3455           $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
3456           if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
3457             $mulValues = explode(',', $val);
3458             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
3459             $newValues[$key] = [];
3460             foreach ($mulValues as $v1) {
3461               foreach ($customOption as $v2) {
3462                 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
3463                   (strtolower($v2['value']) == strtolower(trim($v1)))
3464                 ) {
3465                   if ($htmlType == 'CheckBox') {
3466                     $newValues[$key][$v2['value']] = 1;
3467                   }
3468                   else {
3469                     $newValues[$key][] = $v2['value'];
3470                   }
3471                 }
3472               }
3473             }
3474           }
3475         }
3476       }
3477       // consider new values
3478       $values = $newValues;
3479     }
3480
3481     $fields['Address'] = $this->getMetadataForEntity('Address');
3482     // @todo this is kinda replicated below....
3483     _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$values['location_type_id']]);
3484
3485     $addressFields = [
3486       'county',
3487       'country',
3488       'state_province',
3489       'supplemental_address_1',
3490       'supplemental_address_2',
3491       'supplemental_address_3',
3492       'StateProvince.name',
3493     ];
3494     foreach (array_keys($customFields) as $customFieldID) {
3495       $addressFields[] = 'custom_' . $customFieldID;
3496     }
3497
3498     foreach ($addressFields as $field) {
3499       if (array_key_exists($field, $values)) {
3500         if (!array_key_exists('address', $params)) {
3501           $params['address'] = [];
3502         }
3503         $params['address'][$values['location_type_id']][$field] = $values[$field];
3504       }
3505     }
3506
3507     $this->fillPrimary($params['address'][$values['location_type_id']], $values, 'address', CRM_Utils_Array::value('id', $params));
3508     return TRUE;
3509   }
3510
3511   /**
3512    * Get the field metadata for the relevant entity.
3513    *
3514    * @param string $entity
3515    *
3516    * @return array
3517    */
3518   protected function getMetadataForEntity($entity) {
3519     if (!isset($this->fieldMetadata[$entity])) {
3520       $className = "CRM_Core_DAO_$entity";
3521       $this->fieldMetadata[$entity] = $className::fields();
3522     }
3523     return $this->fieldMetadata[$entity];
3524   }
3525
3526   /**
3527    * Fill in the primary location.
3528    *
3529    * If the contact has a primary address we update it. Otherwise
3530    * we add an address of the default location type.
3531    *
3532    * @param array $params
3533    *   Address block parameters
3534    * @param array $values
3535    *   Input values
3536    * @param string $entity
3537    *  - address, email, phone
3538    * @param int|null $contactID
3539    *
3540    * @throws \CiviCRM_API3_Exception
3541    */
3542   protected function fillPrimary(&$params, $values, $entity, $contactID) {
3543     if ($values['location_type_id'] === 'Primary') {
3544       if ($contactID) {
3545         $primary = civicrm_api3($entity, 'get', [
3546           'return' => 'location_type_id',
3547           'contact_id' => $contactID,
3548           'is_primary' => 1,
3549           'sequential' => 1,
3550         ]);
3551       }
3552       $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
3553       $params['location_type_id'] = (int) (isset($primary) && $primary['count']) ? $primary['values'][0]['location_type_id'] : $defaultLocationType->id;
3554       $params['is_primary'] = 1;
3555     }
3556   }
3557
3558   /**
3559    * Get the civicrm_mapping_field appropriate layout for the mapper input.
3560    *
3561    * The input looks something like ['street_address', 1]
3562    * and would be mapped to ['name' => 'street_address', 'location_type_id' =>
3563    * 1]
3564    *
3565    * @param array $fieldMapping
3566    * @param int $mappingID
3567    * @param int $columnNumber
3568    *
3569    * @return array
3570    * @throws \API_Exception
3571    */
3572   public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
3573     $isRelationshipField = preg_match('/\d*_a_b|b_a$/', $fieldMapping[0]);
3574     $fieldName = $isRelationshipField ? $fieldMapping[1] : $fieldMapping[0];
3575     $locationTypeID = NULL;
3576     $possibleLocationField = $isRelationshipField ? 2 : 1;
3577     if ($fieldName !== 'url' && is_numeric($fieldMapping[$possibleLocationField] ?? NULL)) {
3578       $locationTypeID = $fieldMapping[$possibleLocationField];
3579     }
3580     return [
3581       'name' => $fieldName,
3582       'mapping_id' => $mappingID,
3583       'relationship_type_id' => $isRelationshipField ? substr($fieldMapping[0], 0, -4) : NULL,
3584       'relationship_direction' => $isRelationshipField ? substr($fieldMapping[0], -3) : NULL,
3585       'column_number' => $columnNumber,
3586       'contact_type' => $this->getContactType(),
3587       'website_type_id' => $fieldName !== 'url' ? NULL : ($isRelationshipField ? $fieldMapping[2] : $fieldMapping[1]),
3588       'phone_type_id' => $fieldName !== 'phone' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
3589       'im_provider_id' => $fieldName !== 'im' ? NULL : ($isRelationshipField ? $fieldMapping[3] : $fieldMapping[2]),
3590       'location_type_id' => $locationTypeID,
3591     ];
3592   }
3593
3594   /**
3595    * @param array $mappedField
3596    *   Field detail as would be saved in field_mapping table
3597    *   or as returned from getMappingFieldFromMapperInput
3598    *
3599    * @return string
3600    * @throws \API_Exception
3601    */
3602   public function getMappedFieldLabel(array $mappedField): string {
3603     $this->setFieldMetadata();
3604     $title = [];
3605     if ($mappedField['relationship_type_id']) {
3606       $title[] = $this->getRelationshipLabel($mappedField['relationship_type_id'], $mappedField['relationship_direction']);
3607     }
3608     $title[] = $this->getImportableFieldsMetadata()[$mappedField['name']]['title'];
3609     if ($mappedField['location_type_id']) {
3610       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Address', 'location_type_id', $mappedField['location_type_id']);
3611     }
3612     if ($mappedField['website_type_id']) {
3613       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Website', 'website_type_id', $mappedField['website_type_id']);
3614     }
3615     if ($mappedField['phone_type_id']) {
3616       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $mappedField['phone_type_id']);
3617     }
3618     if ($mappedField['im_provider_id']) {
3619       $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $mappedField['provider_id']);
3620     }
3621     return implode(' - ', $title);
3622   }
3623
3624   /**
3625    * Get the relevant label for the relationship.
3626    *
3627    * @param int $id
3628    * @param string $direction
3629    *
3630    * @return string
3631    * @throws \API_Exception
3632    */
3633   protected function getRelationshipLabel(int $id, string $direction): string {
3634     if (empty($this->relationshipLabels[$id . $direction])) {
3635       $this->relationshipLabels[$id . $direction] =
3636       $fieldName = 'label_' . $direction;
3637       $this->relationshipLabels[$id . $direction] = (string) RelationshipType::get(FALSE)
3638         ->addWhere('id', '=', $id)
3639         ->addSelect($fieldName)->execute()->first()[$fieldName];
3640     }
3641     return $this->relationshipLabels[$id . $direction];
3642   }
3643
3644 }