CRM/Import/ImportProcessor.php

   1 <?php
   2
   3 use Civi\Api4\Mapping;
   4 use Civi\Api4\MappingField;
   5
   6 /**
   7  * Class CRM_Import_ImportProcessor.
   8  *
   9  * Import processor class. This is intended to provide a sanitising wrapper around
  10  * the form-oriented import classes. In particular it is intended to provide a clear translation
  11  * between the saved mapping field format and the quick form & parser formats.
  12  *
  13  * In the first instance this is only being used in unit tests but the intent is to migrate
  14  * to it on a trajectory similar to the ExportProcessor so it is not in the tests.
  15  */
  16 class CRM_Import_ImportProcessor {
  17
  18   /**
  19    * An array of fields in the format used in the table civicrm_mapping_field.
  20    *
  21    * @var array
  22    */
  23   protected $mappingFields = [];
  24
  25   /**
  26    * @var array
  27    */
  28   protected $metadata = [];
  29
  30   /**
  31    * Id of the created user job.
  32    *
  33    * @var int
  34    */
  35   protected $userJobID;
  36
  37   /**
  38    * @return int
  39    */
  40   public function getUserJobID(): int {
  41     return $this->userJobID;
  42   }
  43
  44   /**
  45    * @param int $userJobID
  46    */
  47   public function setUserJobID(int $userJobID): void {
  48     $this->userJobID = $userJobID;
  49   }
  50
  51   /**
  52    * Metadata keyed by field title.
  53    *
  54    * @var array
  55    */
  56   protected $metadataByTitle = [];
  57
  58   /**
  59    * Get contact type being imported.
  60    *
  61    * @var string
  62    */
  63   protected $contactType;
  64
  65   /**
  66    * Get contact sub type being imported.
  67    *
  68    * @var string
  69    */
  70   protected $contactSubType;
  71
  72   /**
  73    * Array of valid relationships for the contact type & subtype.
  74    *
  75    * @var array
  76    */
  77   protected $validRelationships = [];
  78
  79   /**
  80    * Name of  the form.
  81    *
  82    * Used for js for quick form.
  83    *
  84    * @var string
  85    */
  86   protected $formName;
  87
  88   /**
  89    * @return string
  90    */
  91   public function getFormName(): string {
  92     return $this->formName;
  93   }
  94
  95   /**
  96    * @param string $formName
  97    */
  98   public function setFormName(string $formName) {
  99     $this->formName = $formName;
 100   }
 101
 102   /**
 103    * @return array
 104    */
 105   public function getValidRelationships(): array {
 106     if (!isset($this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()])) {
 107       //Relationship importables
 108       $relations = CRM_Contact_BAO_Relationship::getContactRelationshipType(
 109         NULL, NULL, NULL, $this->getContactType(),
 110         FALSE, 'label', TRUE, $this->getContactSubType()
 111       );
 112       asort($relations);
 113       $this->setValidRelationships($relations);
 114     }
 115     return $this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()];
 116   }
 117
 118   /**
 119    * @param array $validRelationships
 120    */
 121   public function setValidRelationships(array $validRelationships) {
 122     $this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()] = $validRelationships;
 123   }
 124
 125   /**
 126    * Get contact subtype for import.
 127    *
 128    * @return string
 129    */
 130   public function getContactSubType(): string {
 131     return $this->contactSubType ?? '';
 132   }
 133
 134   /**
 135    * Set contact subtype for import.
 136    *
 137    * @param string $contactSubType
 138    */
 139   public function setContactSubType($contactSubType) {
 140     $this->contactSubType = (string) $contactSubType;
 141   }
 142
 143   /**
 144    * Saved Mapping ID.
 145    *
 146    * @var int
 147    */
 148   protected $mappingID;
 149
 150   /**
 151    * @return array
 152    */
 153   public function getMetadata(): array {
 154     return $this->metadata;
 155   }
 156
 157   /**
 158    * Setting for metadata.
 159    *
 160    * We wrangle the label for custom fields to include the label since the
 161    * metadata  trait presents it in a more 'pure' form but the label is  appended for importing.
 162    *
 163    * @param array $metadata
 164    *
 165    * @throws \CiviCRM_API3_Exception
 166    */
 167   public function setMetadata(array $metadata) {
 168     $fieldDetails = civicrm_api3('CustomField', 'get', [
 169       'return' => ['custom_group_id.title'],
 170       'options' => ['limit' => 0],
 171     ])['values'];
 172     foreach ($metadata as $index => $field) {
 173       if (!empty($field['custom_field_id'])) {
 174         // The 'label' format for import is custom group title :: custom name title
 175         $metadata[$index]['name'] = $index;
 176         $metadata[$index]['title'] .= ' :: ' . $fieldDetails[$field['custom_field_id']]['custom_group_id.title'];
 177       }
 178     }
 179     $this->metadata = $metadata;
 180   }
 181
 182   /**
 183    * @return int
 184    */
 185   public function getMappingID(): int {
 186     return $this->mappingID;
 187   }
 188
 189   /**
 190    * @param int $mappingID
 191    */
 192   public function setMappingID(int $mappingID) {
 193     $this->mappingID = $mappingID;
 194   }
 195
 196   /**
 197    * Get the contact type for the import.
 198    *
 199    * @return string
 200    */
 201   public function getContactType(): string {
 202     return $this->contactType;
 203   }
 204
 205   /**
 206    * @param string $contactType
 207    */
 208   public function setContactType(string $contactType) {
 209     $this->contactType = $contactType;
 210   }
 211
 212   /**
 213    * Set the contact type  according to the constant.
 214    *
 215    * @param int $contactTypeKey
 216    */
 217   public function setContactTypeByConstant($contactTypeKey) {
 218     $constantTypeMap = [
 219       CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 220       CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 221       CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 222     ];
 223     $this->contactType = $constantTypeMap[$contactTypeKey];
 224   }
 225
 226   /**
 227    * Get Mapping Fields.
 228    *
 229    * @return array
 230    *
 231    * @throws \CiviCRM_API3_Exception
 232    */
 233   public function getMappingFields(): array {
 234     if (empty($this->mappingFields) && !empty($this->getMappingID())) {
 235       $this->loadSavedMapping();
 236     }
 237     return $this->mappingFields;
 238   }
 239
 240   /**
 241    * Set mapping fields.
 242    *
 243    * We do a little cleanup here too.
 244    *
 245    * We ensure that column numbers are set and that the fields are ordered by them.
 246    *
 247    * This would mean the fields could be loaded unsorted.
 248    *
 249    * @param array $mappingFields
 250    */
 251   public function setMappingFields(array $mappingFields) {
 252     $i = 0;
 253     foreach ($mappingFields as &$mappingField) {
 254       if (!isset($mappingField['column_number'])) {
 255         $mappingField['column_number'] = $i;
 256       }
 257       if ($mappingField['column_number'] > $i) {
 258         $i = $mappingField['column_number'];
 259       }
 260       $i++;
 261     }
 262     $this->mappingFields = $this->rekeyBySortedColumnNumbers($mappingFields);
 263   }
 264
 265   /**
 266    * Get the names of the mapped fields.
 267    *
 268    * @throws \CiviCRM_API3_Exception
 269    */
 270   public function getFieldNames() {
 271     return CRM_Utils_Array::collect('name', $this->getMappingFields());
 272   }
 273
 274   /**
 275    * Get the field name for the given column.
 276    *
 277    * @param int $columnNumber
 278    *
 279    * @return string
 280    * @throws \CiviCRM_API3_Exception
 281    */
 282   public function getFieldName($columnNumber) {
 283     return $this->getFieldNames()[$columnNumber];
 284   }
 285
 286   /**
 287    * Get the field name for the given column.
 288    *
 289    * @param int $columnNumber
 290    *
 291    * @return string
 292    * @throws \CiviCRM_API3_Exception
 293    */
 294   public function getRelationshipKey($columnNumber) {
 295     $field = $this->getMappingFields()[$columnNumber];
 296     return empty($field['relationship_type_id']) ? NULL : $field['relationship_type_id'] . '_' . $field['relationship_direction'];
 297   }
 298
 299   /**
 300    * Get relationship key only if it is valid.
 301    *
 302    * @param int $columnNumber
 303    *
 304    * @return string|null
 305    *
 306    * @throws \CiviCRM_API3_Exception
 307    */
 308   public function getValidRelationshipKey($columnNumber) {
 309     $key = $this->getRelationshipKey($columnNumber);
 310     return $this->isValidRelationshipKey($key) ? $key : NULL;
 311   }
 312
 313   /**
 314    * Get the IM Provider ID.
 315    *
 316    * @param int $columnNumber
 317    *
 318    * @return int
 319    *
 320    * @throws \CiviCRM_API3_Exception
 321    */
 322   public function getIMProviderID($columnNumber) {
 323     return $this->getMappingFields()[$columnNumber]['im_provider_id'] ?? NULL;
 324   }
 325
 326   /**
 327    * Get the Phone Type
 328    *
 329    * @param int $columnNumber
 330    *
 331    * @return int
 332    *
 333    * @throws \CiviCRM_API3_Exception
 334    */
 335   public function getPhoneTypeID($columnNumber) {
 336     return $this->getMappingFields()[$columnNumber]['phone_type_id'] ?? NULL;
 337   }
 338
 339   /**
 340    * Get the Website Type
 341    *
 342    * @param int $columnNumber
 343    *
 344    * @return int
 345    *
 346    * @throws \CiviCRM_API3_Exception
 347    */
 348   public function getWebsiteTypeID($columnNumber) {
 349     return $this->getMappingFields()[$columnNumber]['website_type_id'] ?? NULL;
 350   }
 351
 352   /**
 353    * Get the Location Type
 354    *
 355    * Returning 0 rather than null is historical.
 356    *
 357    * @param int $columnNumber
 358    *
 359    * @return int
 360    *
 361    * @throws \CiviCRM_API3_Exception
 362    */
 363   public function getLocationTypeID($columnNumber) {
 364     return $this->getMappingFields()[$columnNumber]['location_type_id'] ?? 0;
 365   }
 366
 367   /**
 368    * Get the IM or Phone type.
 369    *
 370    * We have a field that would be the 'relevant' type - which could be either.
 371    *
 372    * @param int $columnNumber
 373    *
 374    * @return int
 375    *
 376    * @throws \CiviCRM_API3_Exception
 377    */
 378   public function getPhoneOrIMTypeID($columnNumber) {
 379     return $this->getIMProviderID($columnNumber) ?? $this->getPhoneTypeID($columnNumber);
 380   }
 381
 382   /**
 383    * Get the location types of the mapped fields.
 384    *
 385    * @throws \CiviCRM_API3_Exception
 386    */
 387   public function getFieldLocationTypes() {
 388     return CRM_Utils_Array::collect('location_type_id', $this->getMappingFields());
 389   }
 390
 391   /**
 392    * Get the phone types of the mapped fields.
 393    *
 394    * @throws \CiviCRM_API3_Exception
 395    */
 396   public function getFieldPhoneTypes() {
 397     return CRM_Utils_Array::collect('phone_type_id', $this->getMappingFields());
 398   }
 399
 400   /**
 401    * Get the names of the im_provider fields.
 402    *
 403    * @throws \CiviCRM_API3_Exception
 404    */
 405   public function getFieldIMProviderTypes() {
 406     return CRM_Utils_Array::collect('im_provider_id', $this->getMappingFields());
 407   }
 408
 409   /**
 410    * Get the names of the website fields.
 411    *
 412    * @throws \CiviCRM_API3_Exception
 413    */
 414   public function getFieldWebsiteTypes() {
 415     return CRM_Utils_Array::collect('im_provider_id', $this->getMappingFields());
 416   }
 417
 418   /**
 419    * Get an instance of the importer object.
 420    *
 421    * @return CRM_Contact_Import_Parser_Contact
 422    *
 423    * @throws \CiviCRM_API3_Exception
 424    */
 425   public function getImporterObject() {
 426     $importer = new CRM_Contact_Import_Parser_Contact($this->getFieldNames());
 427     $importer->setUserJobID($this->getUserJobID());
 428     $importer->init();
 429     return $importer;
 430   }
 431
 432   /**
 433    * Load the mapping from the datbase into the format that would be received from the UI.
 434    *
 435    * @throws \CiviCRM_API3_Exception
 436    */
 437   protected function loadSavedMapping() {
 438     $fields = civicrm_api3('MappingField', 'get', [
 439       'mapping_id' => $this->getMappingID(),
 440       'options' => ['limit' => 0],
 441     ])['values'];
 442     $skipped = [];
 443     foreach ($fields as $index => $field) {
 444       if (!$this->isValidField($field['name'])) {
 445         // This scenario could occur if the name of a saved mapping field
 446         // changed or became unavailable https://lab.civicrm.org/dev/core/-/issues/3511.
 447         $skipped[] = $field['name'];
 448         $fields[$index]['name'] = $field['name'] = 'do_not_import';
 449       }
 450       $fieldSpec = $this->getFieldMetadata($field['name']);
 451       $fields[$index]['label'] = $fieldSpec['title'];
 452       if (empty($field['location_type_id']) && !empty($fieldSpec['hasLocationType'])) {
 453         $fields[$index]['location_type_id'] = 'Primary';
 454       }
 455     }
 456     if (!empty($skipped)) {
 457       CRM_Core_Session::setStatus(ts('Invalid saved mappings were skipped') . ':' . implode(', ', $skipped));
 458     }
 459     $this->mappingFields = $this->rekeyBySortedColumnNumbers($fields);
 460   }
 461
 462   /**
 463    * Get the metadata for the field.
 464    *
 465    * @param string $fieldName
 466    *
 467    * @return array
 468    */
 469   protected function getFieldMetadata(string $fieldName): array {
 470     return $this->getMetadata()[$fieldName] ?? CRM_Contact_BAO_Contact::importableFields('All')[$fieldName];
 471   }
 472
 473   /**
 474    * Is the field valid for this import.
 475    *
 476    * If not defined in metadata is is not valid.
 477    *
 478    * @param string $fieldName
 479    *
 480    * @return bool
 481    */
 482   public function isValidField(string $fieldName): bool {
 483     return isset($this->getMetadata()[$fieldName]) || isset(CRM_Contact_BAO_Contact::importableFields('All')[$fieldName]);
 484   }
 485
 486   /**
 487    * Load the mapping from the database into the pre-5.50 format.
 488    *
 489    * This is preserved as a copy the upgrade script can use - since the
 490    * upgrade allows the other to be 'fixed'.
 491    *
 492    * @throws \CiviCRM_API3_Exception
 493    */
 494   protected function legacyLoadSavedMapping() {
 495     $fields = civicrm_api3('MappingField', 'get', [
 496       'mapping_id' => $this->getMappingID(),
 497       'options' => ['limit' => 0],
 498     ])['values'];
 499     foreach ($fields as $index => $field) {
 500       // Fix up the fact that for lost reasons we save by label not name.
 501       $fields[$index]['label'] = $field['name'];
 502       if (empty($field['relationship_type_id'])) {
 503         $fields[$index]['name'] = $this->getNameFromLabel($field['name']);
 504       }
 505       else {
 506         // Honour legacy chaos factor.
 507         if ($field['name'] === ts('- do not import -')) {
 508           // This is why we save names not labels people....
 509           $field['name'] = 'do_not_import';
 510         }
 511         $fields[$index]['name'] = strtolower(str_replace(" ", "_", $field['name']));
 512         // fix for edge cases, CRM-4954
 513         if ($fields[$index]['name'] === 'image_url') {
 514           $fields[$index]['name'] = str_replace('url', 'URL', $fields[$index]['name']);
 515         }
 516       }
 517       $fieldSpec = $this->getMetadata()[$fields[$index]['name']];
 518       if (empty($field['location_type_id']) && !empty($fieldSpec['hasLocationType'])) {
 519         $fields[$index]['location_type_id'] = 'Primary';
 520       }
 521     }
 522     $this->mappingFields = $this->rekeyBySortedColumnNumbers($fields);
 523   }
 524
 525   /**
 526    * Get the titles from metadata.
 527    */
 528   public function getMetadataTitles() {
 529     if (empty($this->metadataByTitle)) {
 530       $this->metadataByTitle = CRM_Utils_Array::collect('title', $this->getMetadata());
 531     }
 532     return $this->metadataByTitle;
 533   }
 534
 535   /**
 536    * Rekey the array by the column_number.
 537    *
 538    * @param array $mappingFields
 539    *
 540    * @return array
 541    */
 542   protected function rekeyBySortedColumnNumbers(array $mappingFields) {
 543     $this->mappingFields = CRM_Utils_Array::rekey($mappingFields, 'column_number');
 544     ksort($this->mappingFields);
 545     return $this->mappingFields;
 546   }
 547
 548   /**
 549    * Get the field name from the label.
 550    *
 551    * @param string $label
 552    *
 553    * @return string
 554    */
 555   protected function getNameFromLabel($label) {
 556     $titleMap = array_flip($this->getMetadataTitles());
 557     $label = str_replace(' (match to contact)', '', $label);
 558     return $titleMap[$label] ?? '';
 559   }
 560
 561   /**
 562    * Validate the key against the relationships available for the contatct type & subtype.
 563    *
 564    * @param string $key
 565    *
 566    * @return bool
 567    */
 568   protected function isValidRelationshipKey($key) {
 569     return !empty($this->getValidRelationships()[$key]);
 570   }
 571
 572   /**
 573    * Get the defaults for the column from the saved mapping.
 574    *
 575    * @param int $column
 576    *
 577    * @return array
 578    * @throws \CiviCRM_API3_Exception
 579    */
 580   public function getSavedQuickformDefaultsForColumn($column) {
 581     $fieldMapping = [];
 582
 583     // $sel1 is either unmapped, a relationship or a target field.
 584     if ($this->getFieldName($column) === 'do_not_import') {
 585       return $fieldMapping;
 586     }
 587
 588     if ($this->getValidRelationshipKey($column)) {
 589       $fieldMapping[] = $this->getValidRelationshipKey($column);
 590     }
 591
 592     // $sel1
 593     $fieldMapping[] = $this->getFieldName($column);
 594
 595     // $sel2
 596     if ($this->getWebsiteTypeID($column)) {
 597       $fieldMapping[] = $this->getWebsiteTypeID($column);
 598     }
 599     elseif ($this->getLocationTypeID($column)) {
 600       $fieldMapping[] = $this->getLocationTypeID($column);
 601     }
 602
 603     // $sel3
 604     if ($this->getPhoneOrIMTypeID($column)) {
 605       $fieldMapping[] = $this->getPhoneOrIMTypeID($column);
 606     }
 607     return $fieldMapping;
 608   }
 609
 610   /**
 611    * This exists for use in the FiveFifty Upgrade
 612    *
 613    * @throws \API_Exception|\CiviCRM_API3_Exception
 614    */
 615   public static function convertSavedFields(): void {
 616     $mappings = Mapping::get(FALSE)
 617       ->setSelect(['id', 'contact_type'])
 618       ->addWhere('mapping_type_id:name', '=', 'Import Contact')
 619       ->execute();
 620
 621     foreach ($mappings as $mapping) {
 622       $processor = new CRM_Import_ImportProcessor();
 623       $processor->setMappingID($mapping['id']);
 624       $processor->setMetadata(CRM_Contact_BAO_Contact::importableFields('All'));
 625       $processor->legacyLoadSavedMapping();;
 626       foreach ($processor->getMappingFields() as $field) {
 627         // The if is mostly precautionary against running this more than once
 628         // - which is common in dev if not live...
 629         if ($field['name']) {
 630           MappingField::update(FALSE)
 631             ->setValues(['name' => $field['name']])
 632             ->addWhere('id', '=', $field['id'])
 633             ->execute();
 634         }
 635       }
 636     }
 637   }
 638
 639 }