CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Import_Parser {
  18   /**
  19    * Settings
  20    */
  21   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  22
  23   /**
  24    * Return codes
  25    */
  26   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  27
  28   /**
  29    * Parser modes
  30    */
  31   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  32
  33   /**
  34    * Codes for duplicate record handling
  35    */
  36   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  37
  38   /**
  39    * Contact types
  40    */
  41   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  42
  43
  44   /**
  45    * Total number of non empty lines
  46    * @var int
  47    */
  48   protected $_totalCount;
  49
  50   /**
  51    * Running total number of valid lines
  52    * @var int
  53    */
  54   protected $_validCount;
  55
  56   /**
  57    * Running total number of invalid rows
  58    * @var int
  59    */
  60   protected $_invalidRowCount;
  61
  62   /**
  63    * Maximum number of non-empty/comment lines to process
  64    *
  65    * @var int
  66    */
  67   protected $_maxLinesToProcess;
  68
  69   /**
  70    * Array of error lines, bounded by MAX_ERROR
  71    * @var array
  72    */
  73   protected $_errors;
  74
  75   /**
  76    * Total number of conflict lines
  77    * @var int
  78    */
  79   protected $_conflictCount;
  80
  81   /**
  82    * Array of conflict lines
  83    * @var array
  84    */
  85   protected $_conflicts;
  86
  87   /**
  88    * Total number of duplicate (from database) lines
  89    * @var int
  90    */
  91   protected $_duplicateCount;
  92
  93   /**
  94    * Array of duplicate lines
  95    * @var array
  96    */
  97   protected $_duplicates;
  98
  99   /**
 100    * Running total number of warnings
 101    * @var int
 102    */
 103   protected $_warningCount;
 104
 105   /**
 106    * Maximum number of warnings to store
 107    * @var int
 108    */
 109   protected $_maxWarningCount = self::MAX_WARNINGS;
 110
 111   /**
 112    * Array of warning lines, bounded by MAX_WARNING
 113    * @var array
 114    */
 115   protected $_warnings;
 116
 117   /**
 118    * Array of all the fields that could potentially be part
 119    * of this import process
 120    * @var array
 121    */
 122   protected $_fields;
 123
 124   /**
 125    * Metadata for all available fields, keyed by unique name.
 126    *
 127    * This is intended to supercede $_fields which uses a special sauce format which
 128    * importableFieldsMetadata uses the standard getfields type format.
 129    *
 130    * @var array
 131    */
 132   protected $importableFieldsMetadata = [];
 133
 134   /**
 135    * Get metadata for all importable fields in std getfields style format.
 136    *
 137    * @return array
 138    */
 139   public function getImportableFieldsMetadata(): array {
 140     return $this->importableFieldsMetadata;
 141   }
 142
 143   /**
 144    * Set metadata for all importable fields in std getfields style format.
 145    *
 146    * @param array $importableFieldsMetadata
 147    */
 148   public function setImportableFieldsMetadata(array $importableFieldsMetadata): void {
 149     $this->importableFieldsMetadata = $importableFieldsMetadata;
 150   }
 151
 152   /**
 153    * Array of the fields that are actually part of the import process
 154    * the position in the array also dictates their position in the import
 155    * file
 156    * @var array
 157    */
 158   protected $_activeFields;
 159
 160   /**
 161    * Cache the count of active fields
 162    *
 163    * @var int
 164    */
 165   protected $_activeFieldCount;
 166
 167   /**
 168    * Cache of preview rows
 169    *
 170    * @var array
 171    */
 172   protected $_rows;
 173
 174   /**
 175    * Filename of error data
 176    *
 177    * @var string
 178    */
 179   protected $_errorFileName;
 180
 181   /**
 182    * Filename of conflict data
 183    *
 184    * @var string
 185    */
 186   protected $_conflictFileName;
 187
 188   /**
 189    * Filename of duplicate data
 190    *
 191    * @var string
 192    */
 193   protected $_duplicateFileName;
 194
 195   /**
 196    * Contact type
 197    *
 198    * @var int
 199    */
 200   public $_contactType;
 201   /**
 202    * Contact sub-type
 203    *
 204    * @var int
 205    */
 206   public $_contactSubType;
 207
 208   /**
 209    * Class constructor.
 210    */
 211   public function __construct() {
 212     $this->_maxLinesToProcess = 0;
 213   }
 214
 215   /**
 216    * Abstract function definitions.
 217    */
 218   abstract protected function init();
 219
 220   /**
 221    * @return mixed
 222    */
 223   abstract protected function fini();
 224
 225   /**
 226    * Map field.
 227    *
 228    * @param array $values
 229    *
 230    * @return mixed
 231    */
 232   abstract protected function mapField(&$values);
 233
 234   /**
 235    * Preview.
 236    *
 237    * @param array $values
 238    *
 239    * @return mixed
 240    */
 241   abstract protected function preview(&$values);
 242
 243   /**
 244    * @param $values
 245    *
 246    * @return mixed
 247    */
 248   abstract protected function summary(&$values);
 249
 250   /**
 251    * @param $onDuplicate
 252    * @param $values
 253    *
 254    * @return mixed
 255    */
 256   abstract protected function import($onDuplicate, &$values);
 257
 258   /**
 259    * Set and validate field values.
 260    *
 261    * @param array $elements
 262    *   array.
 263    * @param $erroneousField
 264    *   reference.
 265    *
 266    * @return int
 267    */
 268   public function setActiveFieldValues($elements, &$erroneousField = NULL) {
 269     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 270     for ($i = 0; $i < $maxCount; $i++) {
 271       $this->_activeFields[$i]->setValue($elements[$i]);
 272     }
 273
 274     // reset all the values that we did not have an equivalent import element
 275     for (; $i < $this->_activeFieldCount; $i++) {
 276       $this->_activeFields[$i]->resetValue();
 277     }
 278
 279     // now validate the fields and return false if error
 280     $valid = self::VALID;
 281     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 282       if (!$this->_activeFields[$i]->validate()) {
 283         // no need to do any more validation
 284         $erroneousField = $i;
 285         $valid = self::ERROR;
 286         break;
 287       }
 288     }
 289     return $valid;
 290   }
 291
 292   /**
 293    * Format the field values for input to the api.
 294    *
 295    * @return array
 296    *   (reference) associative array of name/value pairs
 297    */
 298   public function &getActiveFieldParams() {
 299     $params = [];
 300     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 301       if (isset($this->_activeFields[$i]->_value)
 302         && !isset($params[$this->_activeFields[$i]->_name])
 303         && !isset($this->_activeFields[$i]->_related)
 304       ) {
 305
 306         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 307       }
 308     }
 309     return $params;
 310   }
 311
 312   /**
 313    * Add progress bar to the import process. Calculates time remaining, status etc.
 314    *
 315    * @param $statusID
 316    *   status id of the import process saved in $config->uploadDir.
 317    * @param bool $startImport
 318    *   True when progress bar is to be initiated.
 319    * @param $startTimestamp
 320    *   Initial timestamp when the import was started.
 321    * @param $prevTimestamp
 322    *   Previous timestamp when this function was last called.
 323    * @param $totalRowCount
 324    *   Total number of rows in the import file.
 325    *
 326    * @return NULL|$currTimestamp
 327    */
 328   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 329     $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt";
 330
 331     if ($startImport) {
 332       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 333       //do not force the browser to display the save dialog, CRM-7640
 334       $contents = json_encode([0, $status]);
 335       file_put_contents($statusFile, $contents);
 336     }
 337     else {
 338       $rowCount = $this->_rowCount ?? $this->_lineCount;
 339       $currTimestamp = time();
 340       $time = ($currTimestamp - $prevTimestamp);
 341       $recordsLeft = $totalRowCount - $rowCount;
 342       if ($recordsLeft < 0) {
 343         $recordsLeft = 0;
 344       }
 345       $estimatedTime = ($recordsLeft / 50) * $time;
 346       $estMinutes = floor($estimatedTime / 60);
 347       $timeFormatted = '';
 348       if ($estMinutes > 1) {
 349         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 350         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 351       }
 352       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 353       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 354       $statusMsg = ts('%1 of %2 records - %3 remaining',
 355         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 356       );
 357       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 358       $contents = json_encode([$processedPercent, $status]);
 359
 360       file_put_contents($statusFile, $contents);
 361       return $currTimestamp;
 362     }
 363   }
 364
 365   /**
 366    * @return array
 367    */
 368   public function getSelectValues(): array {
 369     $values = [];
 370     foreach ($this->_fields as $name => $field) {
 371       $values[$name] = $field->_title;
 372     }
 373     return $values;
 374   }
 375
 376   /**
 377    * @return array
 378    */
 379   public function getSelectTypes() {
 380     $values = [];
 381     foreach ($this->_fields as $name => $field) {
 382       if (isset($field->_hasLocationType)) {
 383         $values[$name] = $field->_hasLocationType;
 384       }
 385     }
 386     return $values;
 387   }
 388
 389   /**
 390    * @return array
 391    */
 392   public function getHeaderPatterns() {
 393     $values = [];
 394     foreach ($this->_fields as $name => $field) {
 395       if (isset($field->_headerPattern)) {
 396         $values[$name] = $field->_headerPattern;
 397       }
 398     }
 399     return $values;
 400   }
 401
 402   /**
 403    * @return array
 404    */
 405   public function getDataPatterns() {
 406     $values = [];
 407     foreach ($this->_fields as $name => $field) {
 408       $values[$name] = $field->_dataPattern;
 409     }
 410     return $values;
 411   }
 412
 413   /**
 414    * Remove single-quote enclosures from a value array (row).
 415    *
 416    * @param array $values
 417    * @param string $enclosure
 418    *
 419    * @return void
 420    */
 421   public static function encloseScrub(&$values, $enclosure = "'") {
 422     if (empty($values)) {
 423       return;
 424     }
 425
 426     foreach ($values as $k => $v) {
 427       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 428     }
 429   }
 430
 431   /**
 432    * Setter function.
 433    *
 434    * @param int $max
 435    *
 436    * @return void
 437    */
 438   public function setMaxLinesToProcess($max) {
 439     $this->_maxLinesToProcess = $max;
 440   }
 441
 442   /**
 443    * Determines the file extension based on error code.
 444    *
 445    * @var $type error code constant
 446    * @return string
 447    */
 448   public static function errorFileName($type) {
 449     $fileName = NULL;
 450     if (empty($type)) {
 451       return $fileName;
 452     }
 453
 454     $config = CRM_Core_Config::singleton();
 455     $fileName = $config->uploadDir . "sqlImport";
 456     switch ($type) {
 457       case self::ERROR:
 458         $fileName .= '.errors';
 459         break;
 460
 461       case self::CONFLICT:
 462         $fileName .= '.conflicts';
 463         break;
 464
 465       case self::DUPLICATE:
 466         $fileName .= '.duplicates';
 467         break;
 468
 469       case self::NO_MATCH:
 470         $fileName .= '.mismatch';
 471         break;
 472
 473       case self::UNPARSED_ADDRESS_WARNING:
 474         $fileName .= '.unparsedAddress';
 475         break;
 476     }
 477
 478     return $fileName;
 479   }
 480
 481   /**
 482    * Determines the file name based on error code.
 483    *
 484    * @var $type error code constant
 485    * @return string
 486    */
 487   public static function saveFileName($type) {
 488     $fileName = NULL;
 489     if (empty($type)) {
 490       return $fileName;
 491     }
 492     switch ($type) {
 493       case self::ERROR:
 494         $fileName = 'Import_Errors.csv';
 495         break;
 496
 497       case self::CONFLICT:
 498         $fileName = 'Import_Conflicts.csv';
 499         break;
 500
 501       case self::DUPLICATE:
 502         $fileName = 'Import_Duplicates.csv';
 503         break;
 504
 505       case self::NO_MATCH:
 506         $fileName = 'Import_Mismatch.csv';
 507         break;
 508
 509       case self::UNPARSED_ADDRESS_WARNING:
 510         $fileName = 'Import_Unparsed_Address.csv';
 511         break;
 512     }
 513
 514     return $fileName;
 515   }
 516
 517   /**
 518    * Check if contact is a duplicate .
 519    *
 520    * @param array $formatValues
 521    *
 522    * @return array
 523    */
 524   protected function checkContactDuplicate(&$formatValues) {
 525     //retrieve contact id using contact dedupe rule
 526     $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType;
 527     $formatValues['version'] = 3;
 528     require_once 'CRM/Utils/DeprecatedUtils.php';
 529     $params = $formatValues;
 530     static $cIndieFields = NULL;
 531     static $defaultLocationId = NULL;
 532
 533     $contactType = $params['contact_type'];
 534     if ($cIndieFields == NULL) {
 535       $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType);
 536       $cIndieFields = $cTempIndieFields;
 537
 538       $defaultLocation = CRM_Core_BAO_LocationType::getDefault();
 539
 540       // set the value to default location id else set to 1
 541       if (!$defaultLocationId = (int) $defaultLocation->id) {
 542         $defaultLocationId = 1;
 543       }
 544     }
 545
 546     $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields;
 547
 548     $contactFormatted = [];
 549     foreach ($params as $key => $field) {
 550       if ($field == NULL || $field === '') {
 551         continue;
 552       }
 553       // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
 554       // instead of soft credit contact.
 555       if (is_array($field) && $key != "soft_credit") {
 556         foreach ($field as $value) {
 557           $break = FALSE;
 558           if (is_array($value)) {
 559             foreach ($value as $name => $testForEmpty) {
 560               if ($name !== 'phone_type' &&
 561                 ($testForEmpty === '' || $testForEmpty == NULL)
 562               ) {
 563                 $break = TRUE;
 564                 break;
 565               }
 566             }
 567           }
 568           else {
 569             $break = TRUE;
 570           }
 571           if (!$break) {
 572             _civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 573           }
 574         }
 575         continue;
 576       }
 577
 578       $value = [$key => $field];
 579
 580       // check if location related field, then we need to add primary location type
 581       if (in_array($key, $locationFields)) {
 582         $value['location_type_id'] = $defaultLocationId;
 583       }
 584       elseif (array_key_exists($key, $cIndieFields)) {
 585         $value['contact_type'] = $contactType;
 586       }
 587
 588       _civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
 589     }
 590
 591     $contactFormatted['contact_type'] = $contactType;
 592
 593     return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
 594   }
 595
 596   /**
 597    * Parse a field which could be represented by a label or name value rather than the DB value.
 598    *
 599    * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
 600    *
 601    * but if not available then see if we have a label that can be converted to a name.
 602    *
 603    * @param string|int|null $submittedValue
 604    * @param array $fieldSpec
 605    *   Metadata for the field
 606    *
 607    * @return mixed
 608    */
 609   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
 610     // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
 611     if (!isset($fieldSpec['bao'])) {
 612       return $submittedValue;
 613     }
 614     /* @var \CRM_Core_DAO $bao */
 615     $bao = $fieldSpec['bao'];
 616     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
 617     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
 618     if (isset($nameOptions[$submittedValue])) {
 619       return $submittedValue;
 620     }
 621     if (in_array($submittedValue, $nameOptions)) {
 622       return array_search($submittedValue, $nameOptions, TRUE);
 623     }
 624
 625     $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
 626     if (isset($labelOptions[$submittedValue])) {
 627       return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
 628     }
 629     return '';
 630   }
 631
 632   /**
 633    * This is code extracted from 4 places where this exact snippet was being duplicated.
 634    *
 635    * FIXME: Extracting this was a first step, but there's also
 636    *  1. Inconsistency in the way other select options are handled.
 637    *     Contribution adds handling for Select/Radio/Autocomplete
 638    *     Participant/Activity only handles Select/Radio and misses Autocomplete
 639    *     Membership is missing all of it
 640    *  2. Inconsistency with the way this works vs. how it's implemented in Contact import.
 641    *
 642    * @param $customFieldID
 643    * @param $value
 644    * @param $fieldType
 645    * @return array
 646    */
 647   public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
 648     $mulValues = explode(',', $value);
 649     $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
 650     $values = [];
 651     foreach ($mulValues as $v1) {
 652       foreach ($customOption as $customValueID => $customLabel) {
 653         $customValue = $customLabel['value'];
 654         if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
 655           (strtolower(trim($customValue)) == strtolower(trim($v1)))
 656         ) {
 657           if ($fieldType == 'CheckBox') {
 658             $values[$customValue] = 1;
 659           }
 660           else {
 661             $values[] = $customValue;
 662           }
 663         }
 664       }
 665     }
 666     return $values;
 667   }
 668
 669   /**
 670    * Get the ids of any contacts that match according to the rule.
 671    *
 672    * @param array $formatted
 673    *
 674    * @return array
 675    */
 676   protected function getIdsOfMatchingContacts(array $formatted):array {
 677     // the call to the deprecated function seems to add no value other that to do an additional
 678     // check for the contact_id & type.
 679     $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
 680     if (!CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 681       return [];
 682     }
 683     if (is_array($error['error_message']['params'][0])) {
 684       return $error['error_message']['params'][0];
 685     }
 686     else {
 687       return explode(',', $error['error_message']['params'][0]);
 688     }
 689   }
 690
 691 }