CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33 abstract class CRM_Import_Parser {
  34   /**
  35    * Settings
  36    */
  37   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  38
  39   /**
  40    * Return codes
  41    */
  42   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  43
  44   /**
  45    * Parser modes
  46    */
  47   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  48
  49   /**
  50    * Codes for duplicate record handling
  51    */
  52   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  53
  54   /**
  55    * Contact types
  56    */
  57   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  58
  59
  60   /**
  61    * Total number of non empty lines
  62    * @var int
  63    */
  64   protected $_totalCount;
  65
  66   /**
  67    * Running total number of valid lines
  68    * @var int
  69    */
  70   protected $_validCount;
  71
  72   /**
  73    * Running total number of invalid rows
  74    * @var int
  75    */
  76   protected $_invalidRowCount;
  77
  78   /**
  79    * Maximum number of non-empty/comment lines to process
  80    *
  81    * @var int
  82    */
  83   protected $_maxLinesToProcess;
  84
  85   /**
  86    * Array of error lines, bounded by MAX_ERROR
  87    * @var array
  88    */
  89   protected $_errors;
  90
  91   /**
  92    * Total number of conflict lines
  93    * @var int
  94    */
  95   protected $_conflictCount;
  96
  97   /**
  98    * Array of conflict lines
  99    * @var array
 100    */
 101   protected $_conflicts;
 102
 103   /**
 104    * Total number of duplicate (from database) lines
 105    * @var int
 106    */
 107   protected $_duplicateCount;
 108
 109   /**
 110    * Array of duplicate lines
 111    * @var array
 112    */
 113   protected $_duplicates;
 114
 115   /**
 116    * Running total number of warnings
 117    * @var int
 118    */
 119   protected $_warningCount;
 120
 121   /**
 122    * Maximum number of warnings to store
 123    * @var int
 124    */
 125   protected $_maxWarningCount = self::MAX_WARNINGS;
 126
 127   /**
 128    * Array of warning lines, bounded by MAX_WARNING
 129    * @var array
 130    */
 131   protected $_warnings;
 132
 133   /**
 134    * Array of all the fields that could potentially be part
 135    * of this import process
 136    * @var array
 137    */
 138   protected $_fields;
 139
 140   /**
 141    * Metadata for all available fields, keyed by unique name.
 142    *
 143    * This is intended to supercede $_fields which uses a special sauce format which
 144    * importableFieldsMetadata uses the standard getfields type format.
 145    *
 146    * @var array
 147    */
 148   protected $importableFieldsMetadata = [];
 149
 150   /**
 151    * Get metadata for all importable fields in std getfields style format.
 152    *
 153    * @return array
 154    */
 155   public function getImportableFieldsMetadata(): array {
 156     return $this->importableFieldsMetadata;
 157   }
 158
 159   /**
 160    * Set metadata for all importable fields in std getfields style format.
 161    * @param array $importableFieldsMetadata
 162    */
 163   public function setImportableFieldsMetadata(array $importableFieldsMetadata) {
 164     $this->importableFieldsMetadata = $importableFieldsMetadata;
 165   }
 166
 167   /**
 168    * Array of the fields that are actually part of the import process
 169    * the position in the array also dictates their position in the import
 170    * file
 171    * @var array
 172    */
 173   protected $_activeFields;
 174
 175   /**
 176    * Cache the count of active fields
 177    *
 178    * @var int
 179    */
 180   protected $_activeFieldCount;
 181
 182   /**
 183    * Cache of preview rows
 184    *
 185    * @var array
 186    */
 187   protected $_rows;
 188
 189   /**
 190    * Filename of error data
 191    *
 192    * @var string
 193    */
 194   protected $_errorFileName;
 195
 196   /**
 197    * Filename of conflict data
 198    *
 199    * @var string
 200    */
 201   protected $_conflictFileName;
 202
 203   /**
 204    * Filename of duplicate data
 205    *
 206    * @var string
 207    */
 208   protected $_duplicateFileName;
 209
 210   /**
 211    * Contact type
 212    *
 213    * @var int
 214    */
 215   public $_contactType;
 216   /**
 217    * Contact sub-type
 218    *
 219    * @var int
 220    */
 221   public $_contactSubType;
 222
 223   /**
 224    * Class constructor.
 225    */
 226   public function __construct() {
 227     $this->_maxLinesToProcess = 0;
 228   }
 229
 230   /**
 231    * Abstract function definitions.
 232    */
 233   abstract protected function init();
 234
 235   /**
 236    * @return mixed
 237    */
 238   abstract protected function fini();
 239
 240   /**
 241    * Map field.
 242    *
 243    * @param array $values
 244    *
 245    * @return mixed
 246    */
 247   abstract protected function mapField(&$values);
 248
 249   /**
 250    * Preview.
 251    *
 252    * @param array $values
 253    *
 254    * @return mixed
 255    */
 256   abstract protected function preview(&$values);
 257
 258   /**
 259    * @param $values
 260    *
 261    * @return mixed
 262    */
 263   abstract protected function summary(&$values);
 264
 265   /**
 266    * @param $onDuplicate
 267    * @param $values
 268    *
 269    * @return mixed
 270    */
 271   abstract protected function import($onDuplicate, &$values);
 272
 273   /**
 274    * Set and validate field values.
 275    *
 276    * @param array $elements
 277    *   array.
 278    * @param $erroneousField
 279    *   reference.
 280    *
 281    * @return int
 282    */
 283   public function setActiveFieldValues($elements, &$erroneousField) {
 284     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 285     for ($i = 0; $i < $maxCount; $i++) {
 286       $this->_activeFields[$i]->setValue($elements[$i]);
 287     }
 288
 289     // reset all the values that we did not have an equivalent import element
 290     for (; $i < $this->_activeFieldCount; $i++) {
 291       $this->_activeFields[$i]->resetValue();
 292     }
 293
 294     // now validate the fields and return false if error
 295     $valid = self::VALID;
 296     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 297       if (!$this->_activeFields[$i]->validate()) {
 298         // no need to do any more validation
 299         $erroneousField = $i;
 300         $valid = self::ERROR;
 301         break;
 302       }
 303     }
 304     return $valid;
 305   }
 306
 307   /**
 308    * Format the field values for input to the api.
 309    *
 310    * @return array
 311    *   (reference) associative array of name/value pairs
 312    */
 313   public function &getActiveFieldParams() {
 314     $params = [];
 315     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 316       if (isset($this->_activeFields[$i]->_value)
 317         && !isset($params[$this->_activeFields[$i]->_name])
 318         && !isset($this->_activeFields[$i]->_related)
 319       ) {
 320
 321         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 322       }
 323     }
 324     return $params;
 325   }
 326
 327   /**
 328    * Add progress bar to the import process. Calculates time remaining, status etc.
 329    *
 330    * @param $statusID
 331    *   status id of the import process saved in $config->uploadDir.
 332    * @param bool $startImport
 333    *   True when progress bar is to be initiated.
 334    * @param $startTimestamp
 335    *   Initial timstamp when the import was started.
 336    * @param $prevTimestamp
 337    *   Previous timestamp when this function was last called.
 338    * @param $totalRowCount
 339    *   Total number of rows in the import file.
 340    *
 341    * @return NULL|$currTimestamp
 342    */
 343   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 344     $config = CRM_Core_Config::singleton();
 345     $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
 346
 347     if ($startImport) {
 348       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 349       //do not force the browser to display the save dialog, CRM-7640
 350       $contents = json_encode([0, $status]);
 351       file_put_contents($statusFile, $contents);
 352     }
 353     else {
 354       $rowCount = isset($this->_rowCount) ? $this->_rowCount : $this->_lineCount;
 355       $currTimestamp = time();
 356       $totalTime = ($currTimestamp - $startTimestamp);
 357       $time = ($currTimestamp - $prevTimestamp);
 358       $recordsLeft = $totalRowCount - $rowCount;
 359       if ($recordsLeft < 0) {
 360         $recordsLeft = 0;
 361       }
 362       $estimatedTime = ($recordsLeft / 50) * $time;
 363       $estMinutes = floor($estimatedTime / 60);
 364       $timeFormatted = '';
 365       if ($estMinutes > 1) {
 366         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 367         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 368       }
 369       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 370       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 371       $statusMsg = ts('%1 of %2 records - %3 remaining',
 372         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 373       );
 374       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 375       $contents = json_encode([$processedPercent, $status]);
 376
 377       file_put_contents($statusFile, $contents);
 378       return $currTimestamp;
 379     }
 380   }
 381
 382   /**
 383    * @return array
 384    */
 385   public function getSelectValues() {
 386     $values = [];
 387     foreach ($this->_fields as $name => $field) {
 388       $values[$name] = $field->_title;
 389     }
 390     return $values;
 391   }
 392
 393   /**
 394    * @return array
 395    */
 396   public function getSelectTypes() {
 397     $values = [];
 398     foreach ($this->_fields as $name => $field) {
 399       if (isset($field->_hasLocationType)) {
 400         $values[$name] = $field->_hasLocationType;
 401       }
 402     }
 403     return $values;
 404   }
 405
 406   /**
 407    * @return array
 408    */
 409   public function getHeaderPatterns() {
 410     $values = [];
 411     foreach ($this->_fields as $name => $field) {
 412       if (isset($field->_headerPattern)) {
 413         $values[$name] = $field->_headerPattern;
 414       }
 415     }
 416     return $values;
 417   }
 418
 419   /**
 420    * @return array
 421    */
 422   public function getDataPatterns() {
 423     $values = [];
 424     foreach ($this->_fields as $name => $field) {
 425       $values[$name] = $field->_dataPattern;
 426     }
 427     return $values;
 428   }
 429
 430   /**
 431    * Remove single-quote enclosures from a value array (row).
 432    *
 433    * @param array $values
 434    * @param string $enclosure
 435    *
 436    * @return void
 437    */
 438   public static function encloseScrub(&$values, $enclosure = "'") {
 439     if (empty($values)) {
 440       return;
 441     }
 442
 443     foreach ($values as $k => $v) {
 444       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 445     }
 446   }
 447
 448   /**
 449    * Setter function.
 450    *
 451    * @param int $max
 452    *
 453    * @return void
 454    */
 455   public function setMaxLinesToProcess($max) {
 456     $this->_maxLinesToProcess = $max;
 457   }
 458
 459   /**
 460    * Determines the file extension based on error code.
 461    *
 462    * @var $type error code constant
 463    * @return string
 464    */
 465   public static function errorFileName($type) {
 466     $fileName = NULL;
 467     if (empty($type)) {
 468       return $fileName;
 469     }
 470
 471     $config = CRM_Core_Config::singleton();
 472     $fileName = $config->uploadDir . "sqlImport";
 473     switch ($type) {
 474       case self::ERROR:
 475         $fileName .= '.errors';
 476         break;
 477
 478       case self::CONFLICT:
 479         $fileName .= '.conflicts';
 480         break;
 481
 482       case self::DUPLICATE:
 483         $fileName .= '.duplicates';
 484         break;
 485
 486       case self::NO_MATCH:
 487         $fileName .= '.mismatch';
 488         break;
 489
 490       case self::UNPARSED_ADDRESS_WARNING:
 491         $fileName .= '.unparsedAddress';
 492         break;
 493     }
 494
 495     return $fileName;
 496   }
 497
 498   /**
 499    * Determines the file name based on error code.
 500    *
 501    * @var $type error code constant
 502    * @return string
 503    */
 504   public static function saveFileName($type) {
 505     $fileName = NULL;
 506     if (empty($type)) {
 507       return $fileName;
 508     }
 509     switch ($type) {
 510       case self::ERROR:
 511         $fileName = 'Import_Errors.csv';
 512         break;
 513
 514       case self::CONFLICT:
 515         $fileName = 'Import_Conflicts.csv';
 516         break;
 517
 518       case self::DUPLICATE:
 519         $fileName = 'Import_Duplicates.csv';
 520         break;
 521
 522       case self::NO_MATCH:
 523         $fileName = 'Import_Mismatch.csv';
 524         break;
 525
 526       case self::UNPARSED_ADDRESS_WARNING:
 527         $fileName = 'Import_Unparsed_Address.csv';
 528         break;
 529     }
 530
 531     return $fileName;
 532   }
 533
 534   /**
 535    * Check if contact is a duplicate .
 536    *
 537    * @param array $formatValues
 538    *
 539    * @return array
 540    */
 541   protected function checkContactDuplicate(&$formatValues) {
 542     //retrieve contact id using contact dedupe rule
 543     $formatValues['contact_type'] = $this->_contactType;
 544     $formatValues['version'] = 3;
 545     require_once 'CRM/Utils/DeprecatedUtils.php';
 546     $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
 547     return $error;
 548   }
 549
 550   /**
 551    * Parse a field which could be represented by a label or name value rather than the DB value.
 552    *
 553    * We will try to match name first but if not available then see if we have a label that can be converted to a name.
 554    *
 555    * @param string|int|null $submittedValue
 556    * @param array $fieldSpec
 557    *   Metadata for the field
 558    *
 559    * @return mixed
 560    */
 561   protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
 562     /* @var \CRM_Core_DAO $bao */
 563     $bao = $fieldSpec['bao'];
 564     // For historical reasons use validate as context - ie disabled name matches ARE permitted.
 565     $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
 566     if (!isset($nameOptions[$submittedValue])) {
 567       $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
 568       if (isset($labelOptions[$submittedValue])) {
 569         return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
 570       }
 571     }
 572     return '';
 573   }
 574
 575 }