CRM/Custom/Import/Parser/Api.php

   1 <?php
   2
   3 /**
   4  * Class CRM_Custom_Import_Parser_Api
   5  */
   6 class CRM_Custom_Import_Parser_Api extends CRM_Import_Parser {
   7
   8   protected $_entity = '';
   9   protected $_fields = [];
  10   protected $_requiredFields = [];
  11   protected $_dateFields = [];
  12   protected $_multipleCustomData = '';
  13
  14   /**
  15    * Params for the current entity being prepared for the api.
  16    * @var array
  17    */
  18   protected $_params = [];
  19
  20   protected $_fileName;
  21
  22   /**
  23    * Imported file size.
  24    *
  25    * @var int
  26    */
  27   protected $_fileSize;
  28
  29   /**
  30    * Separator being used
  31    * @var string
  32    */
  33   protected $_separator;
  34
  35   /**
  36    * Total number of lines in file
  37    * @var int
  38    */
  39   protected $_lineCount;
  40
  41   /**
  42    * Whether the file has a column header or not
  43    *
  44    * @var bool
  45    */
  46   protected $_haveColumnHeader;
  47
  48   /**
  49    * Class constructor.
  50    *
  51    * @param array $mapperKeys
  52    * @param null $mapperLocType
  53    * @param null $mapperPhoneType
  54    */
  55   public function __construct(&$mapperKeys = [], $mapperLocType = NULL, $mapperPhoneType = NULL) {
  56     parent::__construct();
  57     $this->_mapperKeys = &$mapperKeys;
  58   }
  59
  60   public function setFields() {
  61     $customGroupID = $this->_multipleCustomData;
  62     $importableFields = $this->getGroupFieldsForImport($customGroupID, $this);
  63     $this->_fields = array_merge([
  64       'do_not_import' => ['title' => ts('- do not import -')],
  65       'contact_id' => ['title' => ts('Contact ID')],
  66       'external_identifier' => ['title' => ts('External Identifier')],
  67     ], $importableFields);
  68   }
  69
  70   /**
  71    * The initializer code, called before the processing
  72    *
  73    * @return void
  74    */
  75   public function init() {
  76     $this->setFields();
  77     $fields = $this->_fields;
  78     $hasLocationType = FALSE;
  79
  80     foreach ($fields as $name => $field) {
  81       $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT);
  82       $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//');
  83       $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//');
  84       $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern'], $hasLocationType);
  85     }
  86     $this->setActiveFields($this->_mapperKeys);
  87   }
  88
  89   /**
  90    * Handle the values in preview mode.
  91    *
  92    * @param array $values
  93    *   The array of values belonging to this line.
  94    *
  95    * @return bool
  96    *   the result of this processing
  97    */
  98   public function preview(&$values) {
  99     return $this->summary($values);
 100   }
 101
 102   /**
 103    * @param array $values
 104    *   The array of values belonging to this line.
 105    *
 106    * @return bool
 107    *   the result of this processing
 108    *   It is called from both the preview & the import actions
 109    *
 110    * @see CRM_Custom_Import_Parser_BaseClass::summary()
 111    */
 112   public function summary(&$values) {
 113     $this->setActiveFieldValues($values);
 114     $errorRequired = FALSE;
 115     $missingField = '';
 116     $this->_params = &$this->getActiveFieldParams();
 117
 118     $this->_updateWithId = FALSE;
 119     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
 120
 121     $this->_params = $this->getActiveFieldParams();
 122     foreach ($this->_requiredFields as $requiredField) {
 123       if (empty($this->_params[$requiredField])) {
 124         $errorRequired = TRUE;
 125         $missingField .= ' ' . $requiredField;
 126         CRM_Contact_Import_Parser_Contact::addToErrorMsg($this->_entity, $requiredField);
 127       }
 128     }
 129
 130     if ($errorRequired) {
 131       array_unshift($values, ts('Missing required field(s) :') . $missingField);
 132       return CRM_Import_Parser::ERROR;
 133     }
 134
 135     $errorMessage = NULL;
 136
 137     $contactType = $this->_contactType ? $this->_contactType : 'Organization';
 138     CRM_Contact_Import_Parser_Contact::isErrorInCustomData($this->_params + ['contact_type' => $contactType], $errorMessage, $this->_contactSubType, NULL);
 139
 140     // pseudoconstants
 141     if ($errorMessage) {
 142       $tempMsg = "Invalid value for field(s) : $errorMessage";
 143       array_unshift($values, $tempMsg);
 144       $errorMessage = NULL;
 145       return CRM_Import_Parser::ERROR;
 146     }
 147     return CRM_Import_Parser::VALID;
 148   }
 149
 150   /**
 151    * Handle the values in import mode.
 152    *
 153    * @param int $onDuplicate
 154    *   The code for what action to take on duplicates.
 155    * @param array $values
 156    *   The array of values belonging to this line.
 157    *
 158    * @return bool
 159    *   the result of this processing
 160    */
 161   public function import($onDuplicate, &$values) {
 162     $response = $this->summary($values);
 163     if ($response != CRM_Import_Parser::VALID) {
 164       return $response;
 165     }
 166
 167     $this->_updateWithId = FALSE;
 168     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
 169
 170     $contactType = $this->_contactType ? $this->_contactType : 'Organization';
 171     $formatted = [
 172       'contact_type' => $contactType,
 173     ];
 174
 175     if (isset($this->_params['external_identifier']) && !isset($this->_params['contact_id'])) {
 176       $checkCid = new CRM_Contact_DAO_Contact();
 177       $checkCid->external_identifier = $this->_params['external_identifier'];
 178       $checkCid->find(TRUE);
 179       $formatted['id'] = $checkCid->id;
 180     }
 181     else {
 182       $formatted['id'] = $this->_params['contact_id'];
 183     }
 184
 185     $this->formatCommonData($this->_params, $formatted);
 186     foreach ($formatted['custom'] as $key => $val) {
 187       $this->_params['custom_' . $key] = $val[-1]['value'];
 188     }
 189     $this->_params['skipRecentView'] = TRUE;
 190     $this->_params['check_permissions'] = TRUE;
 191     $this->_params['entity_id'] = $formatted['id'];
 192     try {
 193       civicrm_api3('custom_value', 'create', $this->_params);
 194     }
 195     catch (CiviCRM_API3_Exception $e) {
 196       $error = $e->getMessage();
 197       array_unshift($values, $error);
 198       return CRM_Import_Parser::ERROR;
 199     }
 200   }
 201
 202   /**
 203    * Adapted from CRM_Contact_Import_Parser_Contact::formatCommonData
 204    *
 205    * TODO: Is this function even necessary? All values get passed to the api anyway.
 206    *
 207    * @param array $params
 208    *   Contain record values.
 209    * @param array $formatted
 210    *   Array of formatted data.
 211    */
 212   private function formatCommonData($params, &$formatted) {
 213
 214     $customFields = CRM_Core_BAO_CustomField::getFields(NULL);
 215
 216     //format date first
 217     $session = CRM_Core_Session::singleton();
 218     $dateType = $session->get("dateTypes");
 219     foreach ($params as $key => $val) {
 220       $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
 221       if ($customFieldID) {
 222         //we should not update Date to null, CRM-4062
 223         if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
 224           //CRM-21267
 225           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
 226         }
 227         elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
 228           if (empty($val) && !is_numeric($val)) {
 229             //retain earlier value when Import mode is `Fill`
 230             unset($params[$key]);
 231           }
 232           else {
 233             $params[$key] = CRM_Utils_String::strtoboolstr($val);
 234           }
 235         }
 236       }
 237     }
 238
 239     //now format custom data.
 240     foreach ($params as $key => $field) {
 241
 242       if ($key == 'id' && isset($field)) {
 243         $formatted[$key] = $field;
 244       }
 245
 246       //Handling Custom Data
 247       if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
 248         array_key_exists($customFieldID, $customFields)
 249       ) {
 250
 251         $extends = $customFields[$customFieldID]['extends'] ?? NULL;
 252         $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
 253         $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
 254         $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
 255
 256         if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
 257           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 258           foreach ($customOption as $customValue) {
 259             $val = $customValue['value'] ?? NULL;
 260             $label = strtolower($customValue['label'] ?? '');
 261             $value = strtolower(trim($formatted[$key]));
 262             if (($value == $label) || ($value == strtolower($val))) {
 263               $params[$key] = $formatted[$key] = $val;
 264             }
 265           }
 266         }
 267         elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
 268           $mulValues = explode(',', $formatted[$key]);
 269           $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 270           $formatted[$key] = [];
 271           $params[$key] = [];
 272           foreach ($mulValues as $v1) {
 273             foreach ($customOption as $v2) {
 274               if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
 275                 (strtolower($v2['value']) == strtolower(trim($v1)))
 276               ) {
 277                 if ($htmlType == 'CheckBox') {
 278                   $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
 279                 }
 280                 else {
 281                   $params[$key][] = $formatted[$key][] = $v2['value'];
 282                 }
 283               }
 284             }
 285           }
 286         }
 287       }
 288     }
 289
 290     if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields)) {
 291       // @todo calling api functions directly is not supported
 292       _civicrm_api3_custom_format_params($params, $formatted, $extends);
 293     }
 294   }
 295
 296   /**
 297    * Set import entity.
 298    * @param string $entity
 299    */
 300   public function setEntity($entity) {
 301     $this->_entity = $entity;
 302     $this->_multipleCustomData = $entity;
 303   }
 304
 305   /**
 306    * Return the field ids and names (with groups) for import purpose.
 307    *
 308    * @param int $id
 309    *   Custom group ID.
 310    *
 311    * @return array
 312    *
 313    */
 314   public function getGroupFieldsForImport($id) {
 315     $importableFields = [];
 316     $params = ['custom_group_id' => $id];
 317     $allFields = civicrm_api3('custom_field', 'get', $params);
 318     $fields = $allFields['values'];
 319     foreach ($fields as $id => $values) {
 320       $datatype = $values['data_type'] ?? NULL;
 321       if ($datatype == 'File') {
 322         continue;
 323       }
 324       /* generate the key for the fields array */
 325       $key = "custom_$id";
 326       $regexp = preg_replace('/[.,;:!?]/', '', CRM_Utils_Array::value(0, $values));
 327       $importableFields[$key] = [
 328         'name' => $key,
 329         'title' => $values['label'] ?? NULL,
 330         'headerPattern' => '/' . preg_quote($regexp, '/') . '/',
 331         'import' => 1,
 332         'custom_field_id' => $id,
 333         'options_per_line' => $values['options_per_line'] ?? NULL,
 334         'data_type' => $values['data_type'] ?? NULL,
 335         'html_type' => $values['html_type'] ?? NULL,
 336         'is_search_range' => $values['is_search_range'] ?? NULL,
 337       ];
 338       if (CRM_Utils_Array::value('html_type', $values) == 'Select Date') {
 339         $importableFields[$key]['date_format'] = $values['date_format'] ?? NULL;
 340         $importableFields[$key]['time_format'] = $values['time_format'] ?? NULL;
 341         $this->_dateFields[] = $key;
 342       }
 343     }
 344     return $importableFields;
 345   }
 346
 347   /**
 348    * @param string $fileName
 349    * @param string $separator
 350    * @param int $mapper
 351    * @param bool $skipColumnHeader
 352    * @param int|string $mode
 353    * @param int|string $contactType
 354    * @param int $onDuplicate
 355    *
 356    * @return mixed
 357    * @throws Exception
 358    */
 359   public function run(
 360     $fileName,
 361     $separator,
 362     $mapper,
 363     $skipColumnHeader = FALSE,
 364     $mode = self::MODE_PREVIEW,
 365     $contactType = self::CONTACT_INDIVIDUAL,
 366     $onDuplicate = self::DUPLICATE_SKIP
 367   ) {
 368     if (!is_array($fileName)) {
 369       throw new CRM_Core_Exception('Unable to determine import file');
 370     }
 371     $fileName = $fileName['name'];
 372
 373     switch ($contactType) {
 374       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
 375         $this->_contactType = 'Individual';
 376         break;
 377
 378       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
 379         $this->_contactType = 'Household';
 380         break;
 381
 382       case CRM_Import_Parser::CONTACT_ORGANIZATION:
 383         $this->_contactType = 'Organization';
 384     }
 385     $this->init();
 386
 387     $this->_haveColumnHeader = $skipColumnHeader;
 388
 389     $this->_separator = $separator;
 390
 391     $fd = fopen($fileName, "r");
 392     if (!$fd) {
 393       return FALSE;
 394     }
 395
 396     $this->_lineCount = $this->_warningCount = 0;
 397     $this->_invalidRowCount = $this->_validCount = 0;
 398     $this->_totalCount = 0;
 399
 400     $this->_errors = [];
 401     $this->_warnings = [];
 402
 403     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 404
 405     if ($mode == self::MODE_MAPFIELD) {
 406       $this->_rows = [];
 407     }
 408     else {
 409       $this->_activeFieldCount = count($this->_activeFields);
 410     }
 411
 412     while (!feof($fd)) {
 413       $this->_lineCount++;
 414
 415       $values = fgetcsv($fd, 8192, $separator);
 416       if (!$values) {
 417         continue;
 418       }
 419
 420       self::encloseScrub($values);
 421
 422       // skip column header if we're not in mapfield mode
 423       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 424         $skipColumnHeader = FALSE;
 425         continue;
 426       }
 427
 428       /* trim whitespace around the values */
 429
 430       $empty = TRUE;
 431       foreach ($values as $k => $v) {
 432         $values[$k] = trim($v, " \t\r\n");
 433       }
 434
 435       if (CRM_Utils_System::isNull($values)) {
 436         continue;
 437       }
 438
 439       $this->_totalCount++;
 440
 441       if ($mode == self::MODE_MAPFIELD) {
 442         $returnCode = CRM_Import_Parser::VALID;
 443       }
 444       elseif ($mode == self::MODE_PREVIEW) {
 445         $returnCode = $this->preview($values);
 446       }
 447       elseif ($mode == self::MODE_SUMMARY) {
 448         $returnCode = $this->summary($values);
 449       }
 450       elseif ($mode == self::MODE_IMPORT) {
 451         $returnCode = $this->import($onDuplicate, $values);
 452       }
 453       else {
 454         $returnCode = self::ERROR;
 455       }
 456
 457       // note that a line could be valid but still produce a warning
 458       if ($returnCode & self::VALID) {
 459         $this->_validCount++;
 460         if ($mode == self::MODE_MAPFIELD) {
 461           $this->_rows[] = $values;
 462           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 463         }
 464       }
 465
 466       if ($returnCode & self::WARNING) {
 467         $this->_warningCount++;
 468         if ($this->_warningCount < $this->_maxWarningCount) {
 469           $this->_warnings[] = $this->_lineCount;
 470         }
 471       }
 472
 473       if ($returnCode & self::ERROR) {
 474         $this->_invalidRowCount++;
 475         $recordNumber = $this->_lineCount;
 476         if ($this->_haveColumnHeader) {
 477           $recordNumber--;
 478         }
 479         array_unshift($values, $recordNumber);
 480         $this->_errors[] = $values;
 481       }
 482
 483       if ($returnCode & self::DUPLICATE) {
 484         $this->_duplicateCount++;
 485         $recordNumber = $this->_lineCount;
 486         if ($this->_haveColumnHeader) {
 487           $recordNumber--;
 488         }
 489         array_unshift($values, $recordNumber);
 490         $this->_duplicates[] = $values;
 491         if ($onDuplicate != self::DUPLICATE_SKIP) {
 492           $this->_validCount++;
 493         }
 494       }
 495
 496       // if we are done processing the maxNumber of lines, break
 497       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 498         break;
 499       }
 500     }
 501
 502     fclose($fd);
 503
 504     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 505       $customHeaders = $mapper;
 506
 507       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 508       foreach ($customHeaders as $key => $value) {
 509         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 510           $customHeaders[$key] = $customfields[$id][0];
 511         }
 512       }
 513       if ($this->_invalidRowCount) {
 514         // removed view url for invlaid contacts
 515         $headers = array_merge([
 516           ts('Line Number'),
 517           ts('Reason'),
 518         ], $customHeaders);
 519         $this->_errorFileName = self::errorFileName(self::ERROR);
 520         CRM_Contact_Import_Parser_Contact::exportCSV($this->_errorFileName, $headers, $this->_errors);
 521       }
 522
 523       if ($this->_duplicateCount) {
 524         $headers = array_merge([
 525           ts('Line Number'),
 526           ts('View Activity History URL'),
 527         ], $customHeaders);
 528
 529         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 530         CRM_Contact_Import_Parser_Contact::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 531       }
 532     }
 533   }
 534
 535   /**
 536    * Given a list of the importable field keys that the user has selected
 537    * set the active fields array to this list
 538    *
 539    * @param array $fieldKeys mapped array of values
 540    *
 541    * @return void
 542    */
 543   public function setActiveFields($fieldKeys) {
 544     $this->_activeFieldCount = count($fieldKeys);
 545     foreach ($fieldKeys as $key) {
 546       if (empty($this->_fields[$key])) {
 547         $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
 548       }
 549       else {
 550         $this->_activeFields[] = clone($this->_fields[$key]);
 551       }
 552     }
 553   }
 554
 555   /**
 556    * Format the field values for input to the api.
 557    *
 558    * @return array
 559    *   (reference ) associative array of name/value pairs
 560    */
 561   public function &getActiveFieldParams() {
 562     $params = [];
 563     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 564       if (isset($this->_activeFields[$i]->_value)
 565         && !isset($params[$this->_activeFields[$i]->_name])
 566         && !isset($this->_activeFields[$i]->_related)
 567       ) {
 568         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 569       }
 570     }
 571     return $params;
 572   }
 573
 574   /**
 575    * Store parser values.
 576    *
 577    * @param CRM_Core_Session $store
 578    *
 579    * @param int $mode
 580    *
 581    * @return void
 582    */
 583   public function set($store, $mode = self::MODE_SUMMARY) {
 584     $store->set('fileSize', $this->_fileSize);
 585     $store->set('lineCount', $this->_lineCount);
 586     $store->set('separator', $this->_separator);
 587     $store->set('fields', $this->getSelectValues());
 588
 589     $store->set('headerPatterns', $this->getHeaderPatterns());
 590     $store->set('dataPatterns', $this->getDataPatterns());
 591     $store->set('columnCount', $this->_activeFieldCount);
 592     $store->set('_entity', $this->_entity);
 593     $store->set('totalRowCount', $this->_totalCount);
 594     $store->set('validRowCount', $this->_validCount);
 595     $store->set('invalidRowCount', $this->_invalidRowCount);
 596
 597     switch ($this->_contactType) {
 598       case 'Individual':
 599         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 600         break;
 601
 602       case 'Household':
 603         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 604         break;
 605
 606       case 'Organization':
 607         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 608     }
 609
 610     if ($this->_invalidRowCount) {
 611       $store->set('errorsFileName', $this->_errorFileName);
 612     }
 613
 614     if (isset($this->_rows) && !empty($this->_rows)) {
 615       $store->set('dataValues', $this->_rows);
 616     }
 617
 618     if ($mode == self::MODE_IMPORT) {
 619       $store->set('duplicateRowCount', $this->_duplicateCount);
 620       if ($this->_duplicateCount) {
 621         $store->set('duplicatesFileName', $this->_duplicateFileName);
 622       }
 623     }
 624   }
 625
 626   /**
 627    * @param string $name
 628    * @param $title
 629    * @param int $type
 630    * @param string $headerPattern
 631    * @param string $dataPattern
 632    * @param bool $hasLocationType
 633    */
 634   public function addField(
 635     $name, $title, $type = CRM_Utils_Type::T_INT,
 636     $headerPattern = '//', $dataPattern = '//',
 637     $hasLocationType = FALSE
 638   ) {
 639     $this->_fields[$name] = new CRM_Custom_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 640     if (empty($name)) {
 641       $this->_fields['doNotImport'] = new CRM_Custom_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 642     }
 643   }
 644
 645 }