CRM/Contact/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Contact_Import_Parser extends CRM_Import_Parser {
  18
  19   protected $_tableName;
  20
  21   /**
  22    * Total number of lines in file
  23    *
  24    * @var int
  25    */
  26   protected $_rowCount;
  27
  28   /**
  29    * Running total number of un-matched Contacts.
  30    *
  31    * @var int
  32    */
  33   protected $_unMatchCount;
  34
  35   /**
  36    * Array of unmatched lines.
  37    *
  38    * @var array
  39    */
  40   protected $_unMatch;
  41
  42   /**
  43    * Total number of contacts with unparsed addresses
  44    * @var int
  45    */
  46   protected $_unparsedAddressCount;
  47
  48   /**
  49    * Filename of mismatch data
  50    *
  51    * @var string
  52    */
  53   protected $_misMatchFilemName;
  54
  55   protected $_primaryKeyName;
  56   protected $_statusFieldName;
  57
  58   protected $fieldMetadata = [];
  59   /**
  60    * On duplicate
  61    *
  62    * @var int
  63    */
  64   public $_onDuplicate;
  65
  66   /**
  67    * Dedupe rule group id to use if set
  68    *
  69    * @var int
  70    */
  71   public $_dedupeRuleGroupID = NULL;
  72
  73   /**
  74    * Run import.
  75    *
  76    * @param string $tableName
  77    * @param array $mapper
  78    * @param int $mode
  79    * @param int $contactType
  80    * @param string $primaryKeyName
  81    * @param string $statusFieldName
  82    * @param int $onDuplicate
  83    * @param int $statusID
  84    * @param int $totalRowCount
  85    * @param bool $doGeocodeAddress
  86    * @param int $timeout
  87    * @param string $contactSubType
  88    * @param int $dedupeRuleGroupID
  89    *
  90    * @return mixed
  91    */
  92   public function run(
  93     $tableName,
  94     $mapper = [],
  95     $mode = self::MODE_PREVIEW,
  96     $contactType = self::CONTACT_INDIVIDUAL,
  97     $primaryKeyName = '_id',
  98     $statusFieldName = '_status',
  99     $onDuplicate = self::DUPLICATE_SKIP,
 100     $statusID = NULL,
 101     $totalRowCount = NULL,
 102     $doGeocodeAddress = FALSE,
 103     $timeout = CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 104     $contactSubType = NULL,
 105     $dedupeRuleGroupID = NULL
 106   ) {
 107
 108     // TODO: Make the timeout actually work
 109     $this->_onDuplicate = $onDuplicate;
 110     $this->_dedupeRuleGroupID = $dedupeRuleGroupID;
 111
 112     switch ($contactType) {
 113       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
 114         $this->_contactType = 'Individual';
 115         break;
 116
 117       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
 118         $this->_contactType = 'Household';
 119         break;
 120
 121       case CRM_Import_Parser::CONTACT_ORGANIZATION:
 122         $this->_contactType = 'Organization';
 123     }
 124
 125     $this->_contactSubType = $contactSubType;
 126
 127     $this->init();
 128
 129     $this->_rowCount = $this->_warningCount = 0;
 130     $this->_invalidRowCount = $this->_validCount = 0;
 131     $this->_totalCount = $this->_conflictCount = 0;
 132
 133     $this->_errors = [];
 134     $this->_warnings = [];
 135     $this->_conflicts = [];
 136     $this->_unparsedAddresses = [];
 137
 138     $this->_tableName = $tableName;
 139     $this->_primaryKeyName = $primaryKeyName;
 140     $this->_statusFieldName = $statusFieldName;
 141
 142     if ($mode == self::MODE_MAPFIELD) {
 143       $this->_rows = [];
 144     }
 145     else {
 146       $this->_activeFieldCount = count($this->_activeFields);
 147     }
 148
 149     if ($mode == self::MODE_IMPORT) {
 150       //get the key of email field
 151       foreach ($mapper as $key => $value) {
 152         if (strtolower($value) == 'email') {
 153           $emailKey = $key;
 154           break;
 155         }
 156       }
 157     }
 158
 159     if ($statusID) {
 160       $this->progressImport($statusID);
 161       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 162     }
 163     // get the contents of the temp. import table
 164     $query = "SELECT * FROM $tableName";
 165     if ($mode == self::MODE_IMPORT) {
 166       $query .= " WHERE $statusFieldName = 'NEW'";
 167     }
 168
 169     $result = CRM_Core_DAO::executeQuery($query);
 170
 171     while ($result->fetch()) {
 172       $values = array_values($result->toArray());
 173       $this->_rowCount++;
 174
 175       /* trim whitespace around the values */
 176       foreach ($values as $k => $v) {
 177         $values[$k] = trim($v, " \t\r\n");
 178       }
 179       if (CRM_Utils_System::isNull($values)) {
 180         continue;
 181       }
 182
 183       $this->_totalCount++;
 184
 185       if ($mode == self::MODE_MAPFIELD) {
 186         $returnCode = $this->mapField($values);
 187       }
 188       elseif ($mode == self::MODE_PREVIEW) {
 189         $returnCode = $this->preview($values);
 190       }
 191       elseif ($mode == self::MODE_SUMMARY) {
 192         $returnCode = $this->summary($values);
 193       }
 194       elseif ($mode == self::MODE_IMPORT) {
 195         //print "Running parser in import mode<br/>\n";
 196         $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
 197         if ($statusID && (($this->_rowCount % 50) == 0)) {
 198           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 199         }
 200       }
 201       else {
 202         $returnCode = self::ERROR;
 203       }
 204
 205       // note that a line could be valid but still produce a warning
 206       if ($returnCode & self::VALID) {
 207         $this->_validCount++;
 208         if ($mode == self::MODE_MAPFIELD) {
 209           $this->_rows[] = $values;
 210           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 211         }
 212       }
 213
 214       if ($returnCode & self::WARNING) {
 215         $this->_warningCount++;
 216         if ($this->_warningCount < $this->_maxWarningCount) {
 217           $this->_warningCount[] = $line;
 218         }
 219       }
 220
 221       if ($returnCode & self::ERROR) {
 222         $this->_invalidRowCount++;
 223         array_unshift($values, $this->_rowCount);
 224         $this->_errors[] = $values;
 225       }
 226
 227       if ($returnCode & self::CONFLICT) {
 228         $this->_conflictCount++;
 229         array_unshift($values, $this->_rowCount);
 230         $this->_conflicts[] = $values;
 231       }
 232
 233       if ($returnCode & self::NO_MATCH) {
 234         $this->_unMatchCount++;
 235         array_unshift($values, $this->_rowCount);
 236         $this->_unMatch[] = $values;
 237       }
 238
 239       if ($returnCode & self::DUPLICATE) {
 240         if ($returnCode & self::MULTIPLE_DUPE) {
 241           /* TODO: multi-dupes should be counted apart from singles
 242            * on non-skip action */
 243         }
 244         $this->_duplicateCount++;
 245         array_unshift($values, $this->_rowCount);
 246         $this->_duplicates[] = $values;
 247         if ($onDuplicate != self::DUPLICATE_SKIP) {
 248           $this->_validCount++;
 249         }
 250       }
 251
 252       if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
 253         $this->_unparsedAddressCount++;
 254         array_unshift($values, $this->_rowCount);
 255         $this->_unparsedAddresses[] = $values;
 256       }
 257       // we give the derived class a way of aborting the process
 258       // note that the return code could be multiple code or'ed together
 259       if ($returnCode & self::STOP) {
 260         break;
 261       }
 262
 263       // if we are done processing the maxNumber of lines, break
 264       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 265         break;
 266       }
 267
 268       // see if we've hit our timeout yet
 269       /* if ( $the_thing_with_the_stuff ) {
 270       do_something( );
 271       } */
 272     }
 273
 274     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 275       $customHeaders = $mapper;
 276
 277       $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType);
 278       foreach ($customHeaders as $key => $value) {
 279         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 280           $customHeaders[$key] = $customfields[$id][0];
 281         }
 282       }
 283
 284       if ($this->_invalidRowCount) {
 285         // removed view url for invlaid contacts
 286         $headers = array_merge([
 287           ts('Line Number'),
 288           ts('Reason'),
 289         ], $customHeaders);
 290         $this->_errorFileName = self::errorFileName(self::ERROR);
 291         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 292       }
 293       if ($this->_conflictCount) {
 294         $headers = array_merge([
 295           ts('Line Number'),
 296           ts('Reason'),
 297         ], $customHeaders);
 298         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 299         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 300       }
 301       if ($this->_duplicateCount) {
 302         $headers = array_merge([
 303           ts('Line Number'),
 304           ts('View Contact URL'),
 305         ], $customHeaders);
 306
 307         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 308         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 309       }
 310       if ($this->_unMatchCount) {
 311         $headers = array_merge([
 312           ts('Line Number'),
 313           ts('Reason'),
 314         ], $customHeaders);
 315
 316         $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH);
 317         self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch);
 318       }
 319       if ($this->_unparsedAddressCount) {
 320         $headers = array_merge([
 321           ts('Line Number'),
 322           ts('Contact Edit URL'),
 323         ], $customHeaders);
 324         $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING);
 325         self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses);
 326       }
 327     }
 328     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 329     return $this->fini();
 330   }
 331
 332   /**
 333    * Given a list of the importable field keys that the user has selected.
 334    * set the active fields array to this list
 335    *
 336    * @param array $fieldKeys
 337    *   Mapped array of values.
 338    */
 339   public function setActiveFields($fieldKeys) {
 340     $this->_activeFieldCount = count($fieldKeys);
 341     foreach ($fieldKeys as $key) {
 342       if (empty($this->_fields[$key])) {
 343         $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
 344       }
 345       else {
 346         $this->_activeFields[] = clone($this->_fields[$key]);
 347       }
 348     }
 349   }
 350
 351   /**
 352    * @param $elements
 353    */
 354   public function setActiveFieldLocationTypes($elements) {
 355     for ($i = 0; $i < count($elements); $i++) {
 356       $this->_activeFields[$i]->_hasLocationType = $elements[$i];
 357     }
 358   }
 359
 360   /**
 361    * @param $elements
 362    */
 363
 364   /**
 365    * @param $elements
 366    */
 367   public function setActiveFieldPhoneTypes($elements) {
 368     for ($i = 0; $i < count($elements); $i++) {
 369       $this->_activeFields[$i]->_phoneType = $elements[$i];
 370     }
 371   }
 372
 373   /**
 374    * @param $elements
 375    */
 376   public function setActiveFieldWebsiteTypes($elements) {
 377     for ($i = 0; $i < count($elements); $i++) {
 378       $this->_activeFields[$i]->_websiteType = $elements[$i];
 379     }
 380   }
 381
 382   /**
 383    * Set IM Service Provider type fields.
 384    *
 385    * @param array $elements
 386    *   IM service provider type ids.
 387    */
 388   public function setActiveFieldImProviders($elements) {
 389     for ($i = 0; $i < count($elements); $i++) {
 390       $this->_activeFields[$i]->_imProvider = $elements[$i];
 391     }
 392   }
 393
 394   /**
 395    * @param $elements
 396    */
 397   public function setActiveFieldRelated($elements) {
 398     for ($i = 0; $i < count($elements); $i++) {
 399       $this->_activeFields[$i]->_related = $elements[$i];
 400     }
 401   }
 402
 403   /**
 404    * @param $elements
 405    */
 406   public function setActiveFieldRelatedContactType($elements) {
 407     for ($i = 0; $i < count($elements); $i++) {
 408       $this->_activeFields[$i]->_relatedContactType = $elements[$i];
 409     }
 410   }
 411
 412   /**
 413    * @param $elements
 414    */
 415   public function setActiveFieldRelatedContactDetails($elements) {
 416     for ($i = 0; $i < count($elements); $i++) {
 417       $this->_activeFields[$i]->_relatedContactDetails = $elements[$i];
 418     }
 419   }
 420
 421   /**
 422    * @param $elements
 423    */
 424   public function setActiveFieldRelatedContactLocType($elements) {
 425     for ($i = 0; $i < count($elements); $i++) {
 426       $this->_activeFields[$i]->_relatedContactLocType = $elements[$i];
 427     }
 428   }
 429
 430   /**
 431    * Set active field for related contact's phone type.
 432    *
 433    * @param array $elements
 434    */
 435   public function setActiveFieldRelatedContactPhoneType($elements) {
 436     for ($i = 0; $i < count($elements); $i++) {
 437       $this->_activeFields[$i]->_relatedContactPhoneType = $elements[$i];
 438     }
 439   }
 440
 441   /**
 442    * @param $elements
 443    */
 444   public function setActiveFieldRelatedContactWebsiteType($elements) {
 445     for ($i = 0; $i < count($elements); $i++) {
 446       $this->_activeFields[$i]->_relatedContactWebsiteType = $elements[$i];
 447     }
 448   }
 449
 450   /**
 451    * Set IM Service Provider type fields for related contacts.
 452    *
 453    * @param array $elements
 454    *   IM service provider type ids of related contact.
 455    */
 456   public function setActiveFieldRelatedContactImProvider($elements) {
 457     for ($i = 0; $i < count($elements); $i++) {
 458       $this->_activeFields[$i]->_relatedContactImProvider = $elements[$i];
 459     }
 460   }
 461
 462   /**
 463    * Format the field values for input to the api.
 464    *
 465    * @return array
 466    *   (reference ) associative array of name/value pairs
 467    */
 468   public function &getActiveFieldParams() {
 469     $params = [];
 470
 471     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 472       if ($this->_activeFields[$i]->_name == 'do_not_import') {
 473         continue;
 474       }
 475
 476       if (isset($this->_activeFields[$i]->_value)) {
 477         if (isset($this->_activeFields[$i]->_hasLocationType)) {
 478           if (!isset($params[$this->_activeFields[$i]->_name])) {
 479             $params[$this->_activeFields[$i]->_name] = [];
 480           }
 481
 482           $value = [
 483             $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
 484             'location_type_id' => $this->_activeFields[$i]->_hasLocationType,
 485           ];
 486
 487           if (isset($this->_activeFields[$i]->_phoneType)) {
 488             $value['phone_type_id'] = $this->_activeFields[$i]->_phoneType;
 489           }
 490
 491           // get IM service Provider type id
 492           if (isset($this->_activeFields[$i]->_imProvider)) {
 493             $value['provider_id'] = $this->_activeFields[$i]->_imProvider;
 494           }
 495
 496           $params[$this->_activeFields[$i]->_name][] = $value;
 497         }
 498         elseif (isset($this->_activeFields[$i]->_websiteType)) {
 499           $value = [
 500             $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
 501             'website_type_id' => $this->_activeFields[$i]->_websiteType,
 502           ];
 503
 504           $params[$this->_activeFields[$i]->_name][] = $value;
 505         }
 506
 507         if (!isset($params[$this->_activeFields[$i]->_name])) {
 508           if (!isset($this->_activeFields[$i]->_related)) {
 509             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 510           }
 511         }
 512
 513         //minor fix for CRM-4062
 514         if (isset($this->_activeFields[$i]->_related)) {
 515           if (!isset($params[$this->_activeFields[$i]->_related])) {
 516             $params[$this->_activeFields[$i]->_related] = [];
 517           }
 518
 519           if (!isset($params[$this->_activeFields[$i]->_related]['contact_type']) && !empty($this->_activeFields[$i]->_relatedContactType)) {
 520             $params[$this->_activeFields[$i]->_related]['contact_type'] = $this->_activeFields[$i]->_relatedContactType;
 521           }
 522
 523           if (isset($this->_activeFields[$i]->_relatedContactLocType) && !empty($this->_activeFields[$i]->_value)) {
 524             if (!empty($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails]) &&
 525               !is_array($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])
 526             ) {
 527               $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = [];
 528             }
 529             $value = [
 530               $this->_activeFields[$i]->_relatedContactDetails => $this->_activeFields[$i]->_value,
 531               'location_type_id' => $this->_activeFields[$i]->_relatedContactLocType,
 532             ];
 533
 534             if (isset($this->_activeFields[$i]->_relatedContactPhoneType)) {
 535               $value['phone_type_id'] = $this->_activeFields[$i]->_relatedContactPhoneType;
 536             }
 537
 538             // get IM service Provider type id for related contact
 539             if (isset($this->_activeFields[$i]->_relatedContactImProvider)) {
 540               $value['provider_id'] = $this->_activeFields[$i]->_relatedContactImProvider;
 541             }
 542
 543             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = $value;
 544           }
 545           elseif (isset($this->_activeFields[$i]->_relatedContactWebsiteType)) {
 546             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = [
 547               'url' => $this->_activeFields[$i]->_value,
 548               'website_type_id' => $this->_activeFields[$i]->_relatedContactWebsiteType,
 549             ];
 550           }
 551           else {
 552             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = $this->_activeFields[$i]->_value;
 553           }
 554         }
 555       }
 556     }
 557
 558     return $params;
 559   }
 560
 561   /**
 562    * @return array
 563    */
 564   public function getColumnPatterns() {
 565     CRM_Core_Error::deprecatedFunctionWarning('no  longer used- use   CRM_Contact_Import_MetadataTrait');
 566     $values = [];
 567     foreach ($this->_fields as $name => $field) {
 568       $values[$name] = $field->_columnPattern;
 569     }
 570     return $values;
 571   }
 572
 573   /**
 574    * @param string $name
 575    * @param $title
 576    * @param int $type
 577    * @param string $headerPattern
 578    * @param string $dataPattern
 579    * @param bool $hasLocationType
 580    */
 581   public function addField(
 582     $name, $title, $type = CRM_Utils_Type::T_INT,
 583     $headerPattern = '//', $dataPattern = '//',
 584     $hasLocationType = FALSE
 585   ) {
 586     $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 587     if (empty($name)) {
 588       $this->_fields['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 589     }
 590   }
 591
 592   /**
 593    * Store parser values.
 594    *
 595    * @param CRM_Core_Session $store
 596    *
 597    * @param int $mode
 598    */
 599   public function set($store, $mode = self::MODE_SUMMARY) {
 600     $store->set('rowCount', $this->_rowCount);
 601     $store->set('fields', $this->getSelectValues());
 602     $store->set('fieldTypes', $this->getSelectTypes());
 603
 604     $store->set('columnCount', $this->_activeFieldCount);
 605
 606     $store->set('totalRowCount', $this->_totalCount);
 607     $store->set('validRowCount', $this->_validCount);
 608     $store->set('invalidRowCount', $this->_invalidRowCount);
 609     $store->set('conflictRowCount', $this->_conflictCount);
 610     $store->set('unMatchCount', $this->_unMatchCount);
 611
 612     switch ($this->_contactType) {
 613       case 'Individual':
 614         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 615         break;
 616
 617       case 'Household':
 618         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 619         break;
 620
 621       case 'Organization':
 622         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 623     }
 624
 625     if ($this->_invalidRowCount) {
 626       $store->set('errorsFileName', $this->_errorFileName);
 627     }
 628     if ($this->_conflictCount) {
 629       $store->set('conflictsFileName', $this->_conflictFileName);
 630     }
 631     if (isset($this->_rows) && !empty($this->_rows)) {
 632       $store->set('dataValues', $this->_rows);
 633     }
 634
 635     if ($this->_unMatchCount) {
 636       $store->set('mismatchFileName', $this->_misMatchFilemName);
 637     }
 638
 639     if ($mode == self::MODE_IMPORT) {
 640       $store->set('duplicateRowCount', $this->_duplicateCount);
 641       $store->set('unparsedAddressCount', $this->_unparsedAddressCount);
 642       if ($this->_duplicateCount) {
 643         $store->set('duplicatesFileName', $this->_duplicateFileName);
 644       }
 645       if ($this->_unparsedAddressCount) {
 646         $store->set('errorsFileName', $this->_errorFileName);
 647       }
 648     }
 649     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 650   }
 651
 652   /**
 653    * Export data to a CSV file.
 654    *
 655    * @param string $fileName
 656    * @param array $header
 657    * @param array $data
 658    */
 659   public static function exportCSV($fileName, $header, $data) {
 660
 661     if (file_exists($fileName) && !is_writable($fileName)) {
 662       CRM_Core_Error::movedSiteError($fileName);
 663     }
 664     //hack to remove '_status', '_statusMsg' and '_id' from error file
 665     $errorValues = [];
 666     $dbRecordStatus = ['IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW'];
 667     foreach ($data as $rowCount => $rowValues) {
 668       $count = 0;
 669       foreach ($rowValues as $key => $val) {
 670         if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
 671           break;
 672         }
 673         $errorValues[$rowCount][$key] = $val;
 674         $count++;
 675       }
 676     }
 677     $data = $errorValues;
 678
 679     $output = [];
 680     $fd = fopen($fileName, 'w');
 681
 682     foreach ($header as $key => $value) {
 683       $header[$key] = "\"$value\"";
 684     }
 685     $config = CRM_Core_Config::singleton();
 686     $output[] = implode($config->fieldSeparator, $header);
 687
 688     foreach ($data as $datum) {
 689       foreach ($datum as $key => $value) {
 690         $datum[$key] = "\"$value\"";
 691       }
 692       $output[] = implode($config->fieldSeparator, $datum);
 693     }
 694     fwrite($fd, implode("\n", $output));
 695     fclose($fd);
 696   }
 697
 698   /**
 699    * Update the record with PK $id in the import database table.
 700    *
 701    * @param int $id
 702    * @param array $params
 703    */
 704   public function updateImportRecord($id, &$params) {
 705     $statusFieldName = $this->_statusFieldName;
 706     $primaryKeyName = $this->_primaryKeyName;
 707
 708     if ($statusFieldName && $primaryKeyName) {
 709       $dao = new CRM_Core_DAO();
 710       $db = $dao->getDatabaseConnection();
 711
 712       $query = "UPDATE $this->_tableName
 713                       SET    $statusFieldName = ?,
 714                              ${statusFieldName}Msg = ?
 715                       WHERE  $primaryKeyName = ?";
 716       $args = [
 717         $params[$statusFieldName],
 718         CRM_Utils_Array::value("${statusFieldName}Msg", $params),
 719         $id,
 720       ];
 721
 722       //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
 723
 724       $db->query($query, $args);
 725     }
 726   }
 727
 728   /**
 729    * Format common params data to proper format to store.
 730    *
 731    * @param array $params
 732    *   Contain record values.
 733    * @param array $formatted
 734    *   Array of formatted data.
 735    * @param array $contactFields
 736    *   Contact DAO fields.
 737    */
 738   public function formatCommonData($params, &$formatted, &$contactFields) {
 739     $csType = [
 740       CRM_Utils_Array::value('contact_type', $formatted),
 741     ];
 742
 743     //CRM-5125
 744     //add custom fields for contact sub type
 745     if (!empty($this->_contactSubType)) {
 746       $csType = $this->_contactSubType;
 747     }
 748
 749     if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $formatted)) {
 750       $csType = $relCsType;
 751     }
 752
 753     $customFields = CRM_Core_BAO_CustomField::getFields($formatted['contact_type'], FALSE, FALSE, $csType);
 754
 755     $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
 756     $customFields = $customFields + $addressCustomFields;
 757
 758     //if a Custom Email Greeting, Custom Postal Greeting or Custom Addressee is mapped, and no "Greeting / Addressee Type ID" is provided, then automatically set the type = Customized, CRM-4575
 759     $elements = [
 760       'email_greeting_custom' => 'email_greeting',
 761       'postal_greeting_custom' => 'postal_greeting',
 762       'addressee_custom' => 'addressee',
 763     ];
 764     foreach ($elements as $k => $v) {
 765       if (array_key_exists($k, $params) && !(array_key_exists($v, $params))) {
 766         $label = key(CRM_Core_OptionGroup::values($v, TRUE, NULL, NULL, 'AND v.name = "Customized"'));
 767         $params[$v] = $label;
 768       }
 769     }
 770
 771     //format date first
 772     $session = CRM_Core_Session::singleton();
 773     $dateType = $session->get("dateTypes");
 774     foreach ($params as $key => $val) {
 775       $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 776       if ($customFieldID &&
 777         !array_key_exists($customFieldID, $addressCustomFields)
 778       ) {
 779         //we should not update Date to null, CRM-4062
 780         if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
 781           //CRM-21267
 782           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
 783         }
 784         elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
 785           if (empty($val) && !is_numeric($val) && $this->_onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) {
 786             //retain earlier value when Import mode is `Fill`
 787             unset($params[$key]);
 788           }
 789           else {
 790             $params[$key] = CRM_Utils_String::strtoboolstr($val);
 791           }
 792         }
 793       }
 794
 795       if ($key == 'birth_date' && $val) {
 796         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
 797       }
 798       elseif ($key == 'deceased_date' && $val) {
 799         CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
 800         $params['is_deceased'] = 1;
 801       }
 802       elseif ($key == 'is_deceased' && $val) {
 803         $params[$key] = CRM_Utils_String::strtoboolstr($val);
 804       }
 805     }
 806
 807     //now format custom data.
 808     foreach ($params as $key => $field) {
 809       if (is_array($field)) {
 810         $isAddressCustomField = FALSE;
 811         foreach ($field as $value) {
 812           $break = FALSE;
 813           if (is_array($value)) {
 814             foreach ($value as $name => $testForEmpty) {
 815               if ($addressCustomFieldID = CRM_Core_BAO_CustomField::getKeyID($name)) {
 816                 $isAddressCustomField = TRUE;
 817                 break;
 818               }
 819               // check if $value does not contain IM provider or phoneType
 820               if (($name !== 'phone_type_id' || $name !== 'provider_id') && ($testForEmpty === '' || $testForEmpty == NULL)) {
 821                 $break = TRUE;
 822                 break;
 823               }
 824             }
 825           }
 826           else {
 827             $break = TRUE;
 828           }
 829
 830           if (!$break) {
 831             if (!empty($value['location_type_id'])) {
 832               $this->formatLocationBlock($value, $formatted);
 833             }
 834             else {
 835               // @todo - this is still reachable - e.g. import with related contact info like firstname,lastname,spouse-first-name,spouse-last-name,spouse-home-phone
 836               CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
 837               $this->formatContactParameters($value, $formatted);
 838             }
 839           }
 840         }
 841         if (!$isAddressCustomField) {
 842           continue;
 843         }
 844       }
 845
 846       $formatValues = [
 847         $key => $field,
 848       ];
 849
 850       if (($key !== 'preferred_communication_method') && (array_key_exists($key, $contactFields))) {
 851         // due to merging of individual table and
 852         // contact table, we need to avoid
 853         // preferred_communication_method forcefully
 854         $formatValues['contact_type'] = $formatted['contact_type'];
 855       }
 856
 857       if ($key == 'id' && isset($field)) {
 858         $formatted[$key] = $field;
 859       }
 860       $this->formatContactParameters($formatValues, $formatted);
 861
 862       //Handling Custom Data
 863       // note: Address custom fields will be handled separately inside formatContactParameters
 864       if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
 865         array_key_exists($customFieldID, $customFields) &&
 866         !array_key_exists($customFieldID, $addressCustomFields)
 867       ) {
 868
 869         $extends = $customFields[$customFieldID]['extends'] ?? NULL;
 870         $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 871         $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
 872         $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
 873
 874         if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
 875           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 876           foreach ($customOption as $customValue) {
 877             $val = $customValue['value'] ?? NULL;
 878             $label = strtolower($customValue['label'] ?? '');
 879             $value = strtolower(trim($formatted[$key]));
 880             if (($value == $label) || ($value == strtolower($val))) {
 881               $params[$key] = $formatted[$key] = $val;
 882             }
 883           }
 884         }
 885         elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
 886           $mulValues = explode(',', $formatted[$key]);
 887           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 888           $formatted[$key] = [];
 889           $params[$key] = [];
 890           foreach ($mulValues as $v1) {
 891             foreach ($customOption as $v2) {
 892               if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 893                 (strtolower($v2['value']) == strtolower(trim($v1)))
 894               ) {
 895                 if ($htmlType == 'CheckBox') {
 896                   $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
 897                 }
 898                 else {
 899                   $params[$key][] = $formatted[$key][] = $v2['value'];
 900                 }
 901               }
 902             }
 903           }
 904         }
 905       }
 906     }
 907
 908     if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields) &&
 909       !array_key_exists($customFieldID, $addressCustomFields)
 910     ) {
 911       // @todo calling api functions directly is not supported
 912       _civicrm_api3_custom_format_params($params, $formatted, $extends);
 913     }
 914
 915     // to check if not update mode and unset the fields with empty value.
 916     if (!$this->_updateWithId && array_key_exists('custom', $formatted)) {
 917       foreach ($formatted['custom'] as $customKey => $customvalue) {
 918         if (empty($formatted['custom'][$customKey][-1]['is_required'])) {
 919           $formatted['custom'][$customKey][-1]['is_required'] = $customFields[$customKey]['is_required'];
 920         }
 921         $emptyValue = $customvalue[-1]['value'] ?? NULL;
 922         if (!isset($emptyValue)) {
 923           unset($formatted['custom'][$customKey]);
 924         }
 925       }
 926     }
 927
 928     // parse street address, CRM-5450
 929     if ($this->_parseStreetAddress) {
 930       if (array_key_exists('address', $formatted) && is_array($formatted['address'])) {
 931         foreach ($formatted['address'] as $instance => & $address) {
 932           $streetAddress = $address['street_address'] ?? NULL;
 933           if (empty($streetAddress)) {
 934             continue;
 935           }
 936           // parse address field.
 937           $parsedFields = CRM_Core_BAO_Address::parseStreetAddress($streetAddress);
 938
 939           //street address consider to be parsed properly,
 940           //If we get street_name and street_number.
 941           if (empty($parsedFields['street_name']) || empty($parsedFields['street_number'])) {
 942             $parsedFields = array_fill_keys(array_keys($parsedFields), '');
 943           }
 944
 945           // merge parse address w/ main address block.
 946           $address = array_merge($address, $parsedFields);
 947         }
 948       }
 949     }
 950   }
 951
 952   /**
 953    * Format contact parameters.
 954    *
 955    * @todo this function needs re-writing & re-merging into the main function.
 956    *
 957    * Here be dragons.
 958    *
 959    * @param array $values
 960    * @param array $params
 961    *
 962    * @return bool
 963    */
 964   protected function formatContactParameters(&$values, &$params) {
 965     // Crawl through the possible classes:
 966     // Contact
 967     //      Individual
 968     //      Household
 969     //      Organization
 970     //          Location
 971     //              Address
 972     //              Email
 973     //              Phone
 974     //              IM
 975     //      Note
 976     //      Custom
 977
 978     // first add core contact values since for other Civi modules they are not added
 979     $contactFields = CRM_Contact_DAO_Contact::fields();
 980     _civicrm_api3_store_values($contactFields, $values, $params);
 981
 982     if (isset($values['contact_type'])) {
 983       // we're an individual/household/org property
 984
 985       $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
 986
 987       _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
 988       return TRUE;
 989     }
 990
 991     // Cache the various object fields
 992     // @todo - remove this after confirming this is just a compilation of other-wise-cached fields.
 993     static $fields = [];
 994
 995     if (isset($values['individual_prefix'])) {
 996       if (!empty($params['prefix_id'])) {
 997         $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
 998         $params['prefix'] = $prefixes[$params['prefix_id']];
 999       }
1000       else {
1001         $params['prefix'] = $values['individual_prefix'];
1002       }
1003       return TRUE;
1004     }
1005
1006     if (isset($values['individual_suffix'])) {
1007       if (!empty($params['suffix_id'])) {
1008         $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
1009         $params['suffix'] = $suffixes[$params['suffix_id']];
1010       }
1011       else {
1012         $params['suffix'] = $values['individual_suffix'];
1013       }
1014       return TRUE;
1015     }
1016
1017     // CRM-4575
1018     if (isset($values['email_greeting'])) {
1019       if (!empty($params['email_greeting_id'])) {
1020         $emailGreetingFilter = [
1021           'contact_type' => $params['contact_type'] ?? NULL,
1022           'greeting_type' => 'email_greeting',
1023         ];
1024         $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
1025         $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
1026       }
1027       else {
1028         $params['email_greeting'] = $values['email_greeting'];
1029       }
1030
1031       return TRUE;
1032     }
1033
1034     if (isset($values['postal_greeting'])) {
1035       if (!empty($params['postal_greeting_id'])) {
1036         $postalGreetingFilter = [
1037           'contact_type' => $params['contact_type'] ?? NULL,
1038           'greeting_type' => 'postal_greeting',
1039         ];
1040         $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
1041         $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
1042       }
1043       else {
1044         $params['postal_greeting'] = $values['postal_greeting'];
1045       }
1046       return TRUE;
1047     }
1048
1049     if (isset($values['addressee'])) {
1050       if (!empty($params['addressee_id'])) {
1051         $addresseeFilter = [
1052           'contact_type' => $params['contact_type'] ?? NULL,
1053           'greeting_type' => 'addressee',
1054         ];
1055         $addressee = CRM_Core_PseudoConstant::addressee($addresseeFilter);
1056         $params['addressee'] = $addressee[$params['addressee_id']];
1057       }
1058       else {
1059         $params['addressee'] = $values['addressee'];
1060       }
1061       return TRUE;
1062     }
1063
1064     if (isset($values['gender'])) {
1065       if (!empty($params['gender_id'])) {
1066         $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
1067         $params['gender'] = $genders[$params['gender_id']];
1068       }
1069       else {
1070         $params['gender'] = $values['gender'];
1071       }
1072       return TRUE;
1073     }
1074
1075     if (!empty($values['preferred_communication_method'])) {
1076       $comm = [];
1077       $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
1078
1079       $preffComm = explode(',', $values['preferred_communication_method']);
1080       foreach ($preffComm as $v) {
1081         $v = strtolower(trim($v));
1082         if (array_key_exists($v, $pcm)) {
1083           $comm[$pcm[$v]] = 1;
1084         }
1085       }
1086
1087       $params['preferred_communication_method'] = $comm;
1088       return TRUE;
1089     }
1090
1091     // format the website params.
1092     if (!empty($values['url'])) {
1093       static $websiteFields;
1094       if (!is_array($websiteFields)) {
1095         $websiteFields = CRM_Core_DAO_Website::fields();
1096       }
1097       if (!array_key_exists('website', $params) ||
1098         !is_array($params['website'])
1099       ) {
1100         $params['website'] = [];
1101       }
1102
1103       $websiteCount = count($params['website']);
1104       _civicrm_api3_store_values($websiteFields, $values,
1105         $params['website'][++$websiteCount]
1106       );
1107
1108       return TRUE;
1109     }
1110
1111     // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1112     if (!empty($values['location_type_id'])) {
1113       CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
1114       return $this->formatLocationBlock($values, $params);
1115     }
1116
1117     if (isset($values['note'])) {
1118       // add a note field
1119       if (!isset($params['note'])) {
1120         $params['note'] = [];
1121       }
1122       $noteBlock = count($params['note']) + 1;
1123
1124       $params['note'][$noteBlock] = [];
1125       if (!isset($fields['Note'])) {
1126         $fields['Note'] = CRM_Core_DAO_Note::fields();
1127       }
1128
1129       // get the current logged in civicrm user
1130       $session = CRM_Core_Session::singleton();
1131       $userID = $session->get('userID');
1132
1133       if ($userID) {
1134         $values['contact_id'] = $userID;
1135       }
1136
1137       _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1138
1139       return TRUE;
1140     }
1141
1142     // Check for custom field values
1143     $customFields = CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1144       FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1145     );
1146
1147     foreach ($values as $key => $value) {
1148       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1149         // check if it's a valid custom field id
1150
1151         if (!array_key_exists($customFieldID, $customFields)) {
1152           return civicrm_api3_create_error('Invalid custom field ID');
1153         }
1154         else {
1155           $params[$key] = $value;
1156         }
1157       }
1158     }
1159     return TRUE;
1160   }
1161
1162   /**
1163    * Format location block ready for importing.
1164    *
1165    * There is some test coverage for this in CRM_Contact_Import_Parser_ContactTest
1166    * e.g. testImportPrimaryAddress.
1167    *
1168    * @param array $values
1169    * @param array $params
1170    *
1171    * @return bool
1172    */
1173   protected function formatLocationBlock(&$values, &$params) {
1174     $blockTypes = [
1175       'phone' => 'Phone',
1176       'email' => 'Email',
1177       'im' => 'IM',
1178       'openid' => 'OpenID',
1179       'phone_ext' => 'Phone',
1180     ];
1181     foreach ($blockTypes as $blockFieldName => $block) {
1182       if (!array_key_exists($blockFieldName, $values)) {
1183         continue;
1184       }
1185       $blockIndex = $values['location_type_id'] . (!empty($values['phone_type_id']) ? '_' . $values['phone_type_id'] : '');
1186
1187       // block present in value array.
1188       if (!array_key_exists($blockFieldName, $params) || !is_array($params[$blockFieldName])) {
1189         $params[$blockFieldName] = [];
1190       }
1191
1192       $fields[$block] = $this->getMetadataForEntity($block);
1193
1194       // copy value to dao field name.
1195       if ($blockFieldName == 'im') {
1196         $values['name'] = $values[$blockFieldName];
1197       }
1198
1199       _civicrm_api3_store_values($fields[$block], $values,
1200         $params[$blockFieldName][$blockIndex]
1201       );
1202
1203       $this->fillPrimary($params[$blockFieldName][$blockIndex], $values, $block, CRM_Utils_Array::value('id', $params));
1204
1205       if (empty($params['id']) && (count($params[$blockFieldName]) == 1)) {
1206         $params[$blockFieldName][$blockIndex]['is_primary'] = TRUE;
1207       }
1208
1209       // we only process single block at a time.
1210       return TRUE;
1211     }
1212
1213     // handle address fields.
1214     if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1215       $params['address'] = [];
1216     }
1217
1218     // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1219     // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1220     // the address in CRM_Core_BAO_Address::create method
1221     if (!empty($values['location_type_id'])) {
1222       static $customFields = [];
1223       if (empty($customFields)) {
1224         $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1225       }
1226       // make a copy of values, as we going to make changes
1227       $newValues = $values;
1228       foreach ($values as $key => $val) {
1229         $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1230         if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1231
1232           $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1233           if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1234             $mulValues = explode(',', $val);
1235             $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1236             $newValues[$key] = [];
1237             foreach ($mulValues as $v1) {
1238               foreach ($customOption as $v2) {
1239                 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1240                   (strtolower($v2['value']) == strtolower(trim($v1)))
1241                 ) {
1242                   if ($htmlType == 'CheckBox') {
1243                     $newValues[$key][$v2['value']] = 1;
1244                   }
1245                   else {
1246                     $newValues[$key][] = $v2['value'];
1247                   }
1248                 }
1249               }
1250             }
1251           }
1252         }
1253       }
1254       // consider new values
1255       $values = $newValues;
1256     }
1257
1258     $fields['Address'] = $this->getMetadataForEntity('Address');
1259     // @todo this is kinda replicated below....
1260     _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$values['location_type_id']]);
1261
1262     $addressFields = [
1263       'county',
1264       'country',
1265       'state_province',
1266       'supplemental_address_1',
1267       'supplemental_address_2',
1268       'supplemental_address_3',
1269       'StateProvince.name',
1270     ];
1271     foreach (array_keys($customFields) as $customFieldID) {
1272       $addressFields[] = 'custom_' . $customFieldID;
1273     }
1274
1275     foreach ($addressFields as $field) {
1276       if (array_key_exists($field, $values)) {
1277         if (!array_key_exists('address', $params)) {
1278           $params['address'] = [];
1279         }
1280         $params['address'][$values['location_type_id']][$field] = $values[$field];
1281       }
1282     }
1283
1284     $this->fillPrimary($params['address'][$values['location_type_id']], $values, 'address', CRM_Utils_Array::value('id', $params));
1285     return TRUE;
1286   }
1287
1288   /**
1289    * Get the field metadata for the relevant entity.
1290    *
1291    * @param string $entity
1292    *
1293    * @return array
1294    */
1295   protected function getMetadataForEntity($entity) {
1296     if (!isset($this->fieldMetadata[$entity])) {
1297       $className = "CRM_Core_DAO_$entity";
1298       $this->fieldMetadata[$entity] = $className::fields();
1299     }
1300     return $this->fieldMetadata[$entity];
1301   }
1302
1303   /**
1304    * Fill in the primary location.
1305    *
1306    * If the contact has a primary address we update it. Otherwise
1307    * we add an address of the default location type.
1308    *
1309    * @param array $params
1310    *   Address block parameters
1311    * @param array $values
1312    *   Input values
1313    * @param string $entity
1314    *  - address, email, phone
1315    * @param int|null $contactID
1316    *
1317    * @throws \CiviCRM_API3_Exception
1318    */
1319   protected function fillPrimary(&$params, $values, $entity, $contactID) {
1320     if ($values['location_type_id'] === 'Primary') {
1321       if ($contactID) {
1322         $primary = civicrm_api3($entity, 'get', [
1323           'return' => 'location_type_id',
1324           'contact_id' => $contactID,
1325           'is_primary' => 1,
1326           'sequential' => 1,
1327         ]);
1328       }
1329       $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
1330       $params['location_type_id'] = (int) (isset($primary) && $primary['count']) ? $primary['values'][0]['location_type_id'] : $defaultLocationType->id;
1331       $params['is_primary'] = 1;
1332     }
1333   }
1334
1335 }