CRM/Import/ImportProcessor.php

   1 <?php
   2
   3 use Civi\Api4\Mapping;
   4 use Civi\Api4\MappingField;
   5
   6 /**
   7  * Class CRM_Import_ImportProcessor.
   8  *
   9  * Import processor class. This is intended to provide a sanitising wrapper around
  10  * the form-oriented import classes. In particular it is intended to provide a clear translation
  11  * between the saved mapping field format and the quick form & parser formats.
  12  *
  13  * In the first instance this is only being used in unit tests but the intent is to migrate
  14  * to it on a trajectory similar to the ExportProcessor so it is not in the tests.
  15  */
  16 class CRM_Import_ImportProcessor {
  17
  18   /**
  19    * An array of fields in the format used in the table civicrm_mapping_field.
  20    *
  21    * @var array
  22    */
  23   protected $mappingFields = [];
  24
  25   /**
  26    * @var array
  27    */
  28   protected $metadata = [];
  29
  30   /**
  31    * Id of the created user job.
  32    *
  33    * @var int
  34    */
  35   protected $userJobID;
  36
  37   /**
  38    * @return int
  39    */
  40   public function getUserJobID(): int {
  41     return $this->userJobID;
  42   }
  43
  44   /**
  45    * @param int $userJobID
  46    */
  47   public function setUserJobID(int $userJobID): void {
  48     $this->userJobID = $userJobID;
  49   }
  50
  51   /**
  52    * Metadata keyed by field title.
  53    *
  54    * @var array
  55    */
  56   protected $metadataByTitle = [];
  57
  58   /**
  59    * Get contact type being imported.
  60    *
  61    * @var string
  62    */
  63   protected $contactType;
  64
  65   /**
  66    * Get contact sub type being imported.
  67    *
  68    * @var string
  69    */
  70   protected $contactSubType;
  71
  72   /**
  73    * Array of valid relationships for the contact type & subtype.
  74    *
  75    * @var array
  76    */
  77   protected $validRelationships = [];
  78
  79   /**
  80    * Name of  the form.
  81    *
  82    * Used for js for quick form.
  83    *
  84    * @var string
  85    */
  86   protected $formName;
  87
  88   /**
  89    * @return string
  90    */
  91   public function getFormName(): string {
  92     return $this->formName;
  93   }
  94
  95   /**
  96    * @param string $formName
  97    */
  98   public function setFormName(string $formName) {
  99     $this->formName = $formName;
 100   }
 101
 102   /**
 103    * @return array
 104    */
 105   public function getValidRelationships(): array {
 106     if (!isset($this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()])) {
 107       //Relationship importables
 108       $relations = CRM_Contact_BAO_Relationship::getContactRelationshipType(
 109         NULL, NULL, NULL, $this->getContactType(),
 110         FALSE, 'label', TRUE, $this->getContactSubType()
 111       );
 112       asort($relations);
 113       $this->setValidRelationships($relations);
 114     }
 115     return $this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()];
 116   }
 117
 118   /**
 119    * @param array $validRelationships
 120    */
 121   public function setValidRelationships(array $validRelationships) {
 122     $this->validRelationships[$this->getContactType() . '_' . $this->getContactSubType()] = $validRelationships;
 123   }
 124
 125   /**
 126    * Get contact subtype for import.
 127    *
 128    * @return string
 129    */
 130   public function getContactSubType(): string {
 131     return $this->contactSubType ?? '';
 132   }
 133
 134   /**
 135    * Set contact subtype for import.
 136    *
 137    * @param string $contactSubType
 138    */
 139   public function setContactSubType($contactSubType) {
 140     $this->contactSubType = (string) $contactSubType;
 141   }
 142
 143   /**
 144    * Saved Mapping ID.
 145    *
 146    * @var int
 147    */
 148   protected $mappingID;
 149
 150   /**
 151    * @return array
 152    */
 153   public function getMetadata(): array {
 154     return $this->metadata;
 155   }
 156
 157   /**
 158    * Setting for metadata.
 159    *
 160    * We wrangle the label for custom fields to include the label since the
 161    * metadata  trait presents it in a more 'pure' form but the label is  appended for importing.
 162    *
 163    * @param array $metadata
 164    *
 165    * @throws \CiviCRM_API3_Exception
 166    */
 167   public function setMetadata(array $metadata) {
 168     $fieldDetails = civicrm_api3('CustomField', 'get', [
 169       'return' => ['custom_group_id.title'],
 170       'options' => ['limit' => 0],
 171     ])['values'];
 172     foreach ($metadata as $index => $field) {
 173       if (!empty($field['custom_field_id'])) {
 174         // The 'label' format for import is custom group title :: custom name title
 175         $metadata[$index]['name'] = $index;
 176         $metadata[$index]['title'] .= ' :: ' . $fieldDetails[$field['custom_field_id']]['custom_group_id.title'];
 177       }
 178     }
 179     $this->metadata = $metadata;
 180   }
 181
 182   /**
 183    * @return int
 184    */
 185   public function getMappingID(): int {
 186     return $this->mappingID;
 187   }
 188
 189   /**
 190    * @param int $mappingID
 191    */
 192   public function setMappingID(int $mappingID) {
 193     $this->mappingID = $mappingID;
 194   }
 195
 196   /**
 197    * Get the contact type for the import.
 198    *
 199    * @return string
 200    */
 201   public function getContactType(): string {
 202     return $this->contactType;
 203   }
 204
 205   /**
 206    * @param string $contactType
 207    */
 208   public function setContactType(string $contactType) {
 209     $this->contactType = $contactType;
 210   }
 211
 212   /**
 213    * Set the contact type  according to the constant.
 214    *
 215    * @param int $contactTypeKey
 216    */
 217   public function setContactTypeByConstant($contactTypeKey) {
 218     $constantTypeMap = [
 219       CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 220       CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 221       CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 222     ];
 223     $this->contactType = $constantTypeMap[$contactTypeKey];
 224   }
 225
 226   /**
 227    * Get Mapping Fields.
 228    *
 229    * @return array
 230    *
 231    * @throws \CiviCRM_API3_Exception
 232    */
 233   public function getMappingFields(): array {
 234     if (empty($this->mappingFields) && !empty($this->getMappingID())) {
 235       $this->loadSavedMapping();
 236     }
 237     return $this->mappingFields;
 238   }
 239
 240   /**
 241    * Set mapping fields.
 242    *
 243    * We do a little cleanup here too.
 244    *
 245    * We ensure that column numbers are set and that the fields are ordered by them.
 246    *
 247    * This would mean the fields could be loaded unsorted.
 248    *
 249    * @param array $mappingFields
 250    */
 251   public function setMappingFields(array $mappingFields) {
 252     $i = 0;
 253     foreach ($mappingFields as &$mappingField) {
 254       if (!isset($mappingField['column_number'])) {
 255         $mappingField['column_number'] = $i;
 256       }
 257       if ($mappingField['column_number'] > $i) {
 258         $i = $mappingField['column_number'];
 259       }
 260       $i++;
 261     }
 262     $this->mappingFields = $this->rekeyBySortedColumnNumbers($mappingFields);
 263   }
 264
 265   /**
 266    * Get the names of the mapped fields.
 267    *
 268    * @throws \CiviCRM_API3_Exception
 269    */
 270   public function getFieldNames() {
 271     return CRM_Utils_Array::collect('name', $this->getMappingFields());
 272   }
 273
 274   /**
 275    * Get the field name for the given column.
 276    *
 277    * @param int $columnNumber
 278    *
 279    * @return string
 280    * @throws \CiviCRM_API3_Exception
 281    */
 282   public function getFieldName($columnNumber) {
 283     return $this->getFieldNames()[$columnNumber];
 284   }
 285
 286   /**
 287    * Get the field name for the given column.
 288    *
 289    * @param int $columnNumber
 290    *
 291    * @return string
 292    * @throws \CiviCRM_API3_Exception
 293    */
 294   public function getRelationshipKey($columnNumber) {
 295     $field = $this->getMappingFields()[$columnNumber];
 296     return empty($field['relationship_type_id']) ? NULL : $field['relationship_type_id'] . '_' . $field['relationship_direction'];
 297   }
 298
 299   /**
 300    * Get relationship key only if it is valid.
 301    *
 302    * @param int $columnNumber
 303    *
 304    * @return string|null
 305    *
 306    * @throws \CiviCRM_API3_Exception
 307    */
 308   public function getValidRelationshipKey($columnNumber) {
 309     $key = $this->getRelationshipKey($columnNumber);
 310     return $this->isValidRelationshipKey($key) ? $key : NULL;
 311   }
 312
 313   /**
 314    * Get the IM Provider ID.
 315    *
 316    * @param int $columnNumber
 317    *
 318    * @return int
 319    *
 320    * @throws \CiviCRM_API3_Exception
 321    */
 322   public function getIMProviderID($columnNumber) {
 323     return $this->getMappingFields()[$columnNumber]['im_provider_id'] ?? NULL;
 324   }
 325
 326   /**
 327    * Get the Phone Type
 328    *
 329    * @param int $columnNumber
 330    *
 331    * @return int
 332    *
 333    * @throws \CiviCRM_API3_Exception
 334    */
 335   public function getPhoneTypeID($columnNumber) {
 336     return $this->getMappingFields()[$columnNumber]['phone_type_id'] ?? NULL;
 337   }
 338
 339   /**
 340    * Get the Website Type
 341    *
 342    * @param int $columnNumber
 343    *
 344    * @return int
 345    *
 346    * @throws \CiviCRM_API3_Exception
 347    */
 348   public function getWebsiteTypeID($columnNumber) {
 349     return $this->getMappingFields()[$columnNumber]['website_type_id'] ?? NULL;
 350   }
 351
 352   /**
 353    * Get the Location Type
 354    *
 355    * Returning 0 rather than null is historical.
 356    *
 357    * @param int $columnNumber
 358    *
 359    * @return int
 360    *
 361    * @throws \CiviCRM_API3_Exception
 362    */
 363   public function getLocationTypeID($columnNumber) {
 364     return $this->getMappingFields()[$columnNumber]['location_type_id'] ?? 0;
 365   }
 366
 367   /**
 368    * Get the IM or Phone type.
 369    *
 370    * We have a field that would be the 'relevant' type - which could be either.
 371    *
 372    * @param int $columnNumber
 373    *
 374    * @return int
 375    *
 376    * @throws \CiviCRM_API3_Exception
 377    */
 378   public function getPhoneOrIMTypeID($columnNumber) {
 379     return $this->getIMProviderID($columnNumber) ?? $this->getPhoneTypeID($columnNumber);
 380   }
 381
 382   /**
 383    * Get the location types of the mapped fields.
 384    *
 385    * @throws \CiviCRM_API3_Exception
 386    */
 387   public function getFieldLocationTypes() {
 388     return CRM_Utils_Array::collect('location_type_id', $this->getMappingFields());
 389   }
 390
 391   /**
 392    * Get the phone types of the mapped fields.
 393    *
 394    * @throws \CiviCRM_API3_Exception
 395    */
 396   public function getFieldPhoneTypes() {
 397     return CRM_Utils_Array::collect('phone_type_id', $this->getMappingFields());
 398   }
 399
 400   /**
 401    * Get the names of the im_provider fields.
 402    *
 403    * @throws \CiviCRM_API3_Exception
 404    */
 405   public function getFieldIMProviderTypes() {
 406     return CRM_Utils_Array::collect('im_provider_id', $this->getMappingFields());
 407   }
 408
 409   /**
 410    * Get the names of the website fields.
 411    *
 412    * @throws \CiviCRM_API3_Exception
 413    */
 414   public function getFieldWebsiteTypes() {
 415     return CRM_Utils_Array::collect('im_provider_id', $this->getMappingFields());
 416   }
 417
 418   /**
 419    * Get an instance of the importer object.
 420    *
 421    * @return CRM_Contact_Import_Parser_Contact
 422    *
 423    * @throws \CiviCRM_API3_Exception
 424    */
 425   public function getImporterObject() {
 426     $importer = new CRM_Contact_Import_Parser_Contact($this->getFieldNames());
 427     $importer->setUserJobID($this->getUserJobID());
 428     $importer->init();
 429     return $importer;
 430   }
 431
 432   /**
 433    * Load the mapping from the datbase into the format that would be received from the UI.
 434    *
 435    * @throws \CiviCRM_API3_Exception
 436    */
 437   protected function loadSavedMapping() {
 438     $fields = civicrm_api3('MappingField', 'get', [
 439       'mapping_id' => $this->getMappingID(),
 440       'options' => ['limit' => 0],
 441     ])['values'];
 442     foreach ($fields as $index => $field) {
 443       $fieldSpec = $this->getFieldMetadata($field['name']);
 444       $fields[$index]['label'] = $fieldSpec['title'];
 445       if (empty($field['location_type_id']) && !empty($fieldSpec['hasLocationType'])) {
 446         $fields[$index]['location_type_id'] = 'Primary';
 447       }
 448     }
 449     $this->mappingFields = $this->rekeyBySortedColumnNumbers($fields);
 450   }
 451
 452   /**
 453    * Get the metadata for the field.
 454    *
 455    * @param string $fieldName
 456    *
 457    * @return array
 458    */
 459   protected function getFieldMetadata(string $fieldName): array {
 460     return $this->getMetadata()[$fieldName] ?? CRM_Contact_BAO_Contact::importableFields('All')[$fieldName];
 461   }
 462
 463   /**
 464    * Load the mapping from the database into the pre-5.50 format.
 465    *
 466    * This is preserved as a copy the upgrade script can use - since the
 467    * upgrade allows the other to be 'fixed'.
 468    *
 469    * @throws \CiviCRM_API3_Exception
 470    */
 471   protected function legacyLoadSavedMapping() {
 472     $fields = civicrm_api3('MappingField', 'get', [
 473       'mapping_id' => $this->getMappingID(),
 474       'options' => ['limit' => 0],
 475     ])['values'];
 476     foreach ($fields as $index => $field) {
 477       // Fix up the fact that for lost reasons we save by label not name.
 478       $fields[$index]['label'] = $field['name'];
 479       if (empty($field['relationship_type_id'])) {
 480         $fields[$index]['name'] = $this->getNameFromLabel($field['name']);
 481       }
 482       else {
 483         // Honour legacy chaos factor.
 484         if ($field['name'] === ts('- do not import -')) {
 485           // This is why we save names not labels people....
 486           $field['name'] = 'do_not_import';
 487         }
 488         $fields[$index]['name'] = strtolower(str_replace(" ", "_", $field['name']));
 489         // fix for edge cases, CRM-4954
 490         if ($fields[$index]['name'] === 'image_url') {
 491           $fields[$index]['name'] = str_replace('url', 'URL', $fields[$index]['name']);
 492         }
 493       }
 494       $fieldSpec = $this->getMetadata()[$fields[$index]['name']];
 495       if (empty($field['location_type_id']) && !empty($fieldSpec['hasLocationType'])) {
 496         $fields[$index]['location_type_id'] = 'Primary';
 497       }
 498     }
 499     $this->mappingFields = $this->rekeyBySortedColumnNumbers($fields);
 500   }
 501
 502   /**
 503    * Get the titles from metadata.
 504    */
 505   public function getMetadataTitles() {
 506     if (empty($this->metadataByTitle)) {
 507       $this->metadataByTitle = CRM_Utils_Array::collect('title', $this->getMetadata());
 508     }
 509     return $this->metadataByTitle;
 510   }
 511
 512   /**
 513    * Rekey the array by the column_number.
 514    *
 515    * @param array $mappingFields
 516    *
 517    * @return array
 518    */
 519   protected function rekeyBySortedColumnNumbers(array $mappingFields) {
 520     $this->mappingFields = CRM_Utils_Array::rekey($mappingFields, 'column_number');
 521     ksort($this->mappingFields);
 522     return $this->mappingFields;
 523   }
 524
 525   /**
 526    * Get the field name from the label.
 527    *
 528    * @param string $label
 529    *
 530    * @return string
 531    */
 532   protected function getNameFromLabel($label) {
 533     $titleMap = array_flip($this->getMetadataTitles());
 534     $label = str_replace(' (match to contact)', '', $label);
 535     return $titleMap[$label] ?? '';
 536   }
 537
 538   /**
 539    * Validate the key against the relationships available for the contatct type & subtype.
 540    *
 541    * @param string $key
 542    *
 543    * @return bool
 544    */
 545   protected function isValidRelationshipKey($key) {
 546     return !empty($this->getValidRelationships()[$key]);
 547   }
 548
 549   /**
 550    * Get the defaults for the column from the saved mapping.
 551    *
 552    * @param int $column
 553    *
 554    * @return array
 555    * @throws \CiviCRM_API3_Exception
 556    */
 557   public function getSavedQuickformDefaultsForColumn($column) {
 558     $fieldMapping = [];
 559
 560     // $sel1 is either unmapped, a relationship or a target field.
 561     if ($this->getFieldName($column) === 'do_not_import') {
 562       return $fieldMapping;
 563     }
 564
 565     if ($this->getValidRelationshipKey($column)) {
 566       $fieldMapping[] = $this->getValidRelationshipKey($column);
 567     }
 568
 569     // $sel1
 570     $fieldMapping[] = $this->getFieldName($column);
 571
 572     // $sel2
 573     if ($this->getWebsiteTypeID($column)) {
 574       $fieldMapping[] = $this->getWebsiteTypeID($column);
 575     }
 576     elseif ($this->getLocationTypeID($column)) {
 577       $fieldMapping[] = $this->getLocationTypeID($column);
 578     }
 579
 580     // $sel3
 581     if ($this->getPhoneOrIMTypeID($column)) {
 582       $fieldMapping[] = $this->getPhoneOrIMTypeID($column);
 583     }
 584     return $fieldMapping;
 585   }
 586
 587   /**
 588    * This exists for use in the FiveFifty Upgrade
 589    *
 590    * @throws \API_Exception|\CiviCRM_API3_Exception
 591    */
 592   public static function convertSavedFields(): void {
 593     $mappings = Mapping::get(FALSE)
 594       ->setSelect(['id', 'contact_type'])
 595       ->addWhere('mapping_type_id:name', '=', 'Import Contact')
 596       ->execute();
 597
 598     foreach ($mappings as $mapping) {
 599       $processor = new CRM_Import_ImportProcessor();
 600       $processor->setMappingID($mapping['id']);
 601       $processor->setMetadata(CRM_Contact_BAO_Contact::importableFields('All'));
 602       $processor->legacyLoadSavedMapping();;
 603       foreach ($processor->getMappingFields() as $field) {
 604         // The if is mostly precautionary against running this more than once
 605         // - which is common in dev if not live...
 606         if ($field['name']) {
 607           MappingField::update(FALSE)
 608             ->setValues(['name' => $field['name']])
 609             ->addWhere('id', '=', $field['id'])
 610             ->execute();
 611         }
 612       }
 613     }
 614   }
 615
 616 }