CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33 abstract class CRM_Import_Parser {
  34   /**
  35    * Settings
  36    */
  37   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  38
  39   /**
  40    * Return codes
  41    */
  42   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  43
  44   /**
  45    * Parser modes
  46    */
  47   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  48
  49   /**
  50    * Codes for duplicate record handling
  51    */
  52   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  53
  54   /**
  55    * Contact types
  56    */
  57   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  58
  59
  60   /**
  61    * Total number of non empty lines
  62    * @var int
  63    */
  64   protected $_totalCount;
  65
  66   /**
  67    * Running total number of valid lines
  68    * @var int
  69    */
  70   protected $_validCount;
  71
  72   /**
  73    * Running total number of invalid rows
  74    * @var int
  75    */
  76   protected $_invalidRowCount;
  77
  78   /**
  79    * Maximum number of non-empty/comment lines to process
  80    *
  81    * @var int
  82    */
  83   protected $_maxLinesToProcess;
  84
  85   /**
  86    * Array of error lines, bounded by MAX_ERROR
  87    * @var array
  88    */
  89   protected $_errors;
  90
  91   /**
  92    * Total number of conflict lines
  93    * @var int
  94    */
  95   protected $_conflictCount;
  96
  97   /**
  98    * Array of conflict lines
  99    * @var array
 100    */
 101   protected $_conflicts;
 102
 103   /**
 104    * Total number of duplicate (from database) lines
 105    * @var int
 106    */
 107   protected $_duplicateCount;
 108
 109   /**
 110    * Array of duplicate lines
 111    * @var array
 112    */
 113   protected $_duplicates;
 114
 115   /**
 116    * Running total number of warnings
 117    * @var int
 118    */
 119   protected $_warningCount;
 120
 121   /**
 122    * Maximum number of warnings to store
 123    * @var int
 124    */
 125   protected $_maxWarningCount = self::MAX_WARNINGS;
 126
 127   /**
 128    * Array of warning lines, bounded by MAX_WARNING
 129    * @var array
 130    */
 131   protected $_warnings;
 132
 133   /**
 134    * Array of all the fields that could potentially be part
 135    * of this import process
 136    * @var array
 137    */
 138   protected $_fields;
 139
 140   /**
 141    * Array of the fields that are actually part of the import process
 142    * the position in the array also dictates their position in the import
 143    * file
 144    * @var array
 145    */
 146   protected $_activeFields;
 147
 148   /**
 149    * Cache the count of active fields
 150    *
 151    * @var int
 152    */
 153   protected $_activeFieldCount;
 154
 155   /**
 156    * Cache of preview rows
 157    *
 158    * @var array
 159    */
 160   protected $_rows;
 161
 162   /**
 163    * Filename of error data
 164    *
 165    * @var string
 166    */
 167   protected $_errorFileName;
 168
 169   /**
 170    * Filename of conflict data
 171    *
 172    * @var string
 173    */
 174   protected $_conflictFileName;
 175
 176   /**
 177    * Filename of duplicate data
 178    *
 179    * @var string
 180    */
 181   protected $_duplicateFileName;
 182
 183   /**
 184    * Contact type
 185    *
 186    * @var int
 187    */
 188   public $_contactType;
 189   /**
 190    * Contact sub-type
 191    *
 192    * @var int
 193    */
 194   public $_contactSubType;
 195
 196   /**
 197    * Class constructor.
 198    */
 199   public function __construct() {
 200     $this->_maxLinesToProcess = 0;
 201   }
 202
 203   /**
 204    * Abstract function definitions.
 205    */
 206   abstract protected function init();
 207
 208   /**
 209    * @return mixed
 210    */
 211   abstract protected function fini();
 212
 213   /**
 214    * Map field.
 215    *
 216    * @param array $values
 217    *
 218    * @return mixed
 219    */
 220   abstract protected function mapField(&$values);
 221
 222   /**
 223    * Preview.
 224    *
 225    * @param array $values
 226    *
 227    * @return mixed
 228    */
 229   abstract protected function preview(&$values);
 230
 231   /**
 232    * @param $values
 233    *
 234    * @return mixed
 235    */
 236   abstract protected function summary(&$values);
 237
 238   /**
 239    * @param $onDuplicate
 240    * @param $values
 241    *
 242    * @return mixed
 243    */
 244   abstract protected function import($onDuplicate, &$values);
 245
 246   /**
 247    * Set and validate field values.
 248    *
 249    * @param array $elements
 250    *   array.
 251    * @param $erroneousField
 252    *   reference.
 253    *
 254    * @return int
 255    */
 256   public function setActiveFieldValues($elements, &$erroneousField) {
 257     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 258     for ($i = 0; $i < $maxCount; $i++) {
 259       $this->_activeFields[$i]->setValue($elements[$i]);
 260     }
 261
 262     // reset all the values that we did not have an equivalent import element
 263     for (; $i < $this->_activeFieldCount; $i++) {
 264       $this->_activeFields[$i]->resetValue();
 265     }
 266
 267     // now validate the fields and return false if error
 268     $valid = self::VALID;
 269     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 270       if (!$this->_activeFields[$i]->validate()) {
 271         // no need to do any more validation
 272         $erroneousField = $i;
 273         $valid = self::ERROR;
 274         break;
 275       }
 276     }
 277     return $valid;
 278   }
 279
 280   /**
 281    * Format the field values for input to the api.
 282    *
 283    * @return array
 284    *   (reference) associative array of name/value pairs
 285    */
 286   public function &getActiveFieldParams() {
 287     $params = [];
 288     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 289       if (isset($this->_activeFields[$i]->_value)
 290         && !isset($params[$this->_activeFields[$i]->_name])
 291         && !isset($this->_activeFields[$i]->_related)
 292       ) {
 293
 294         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 295       }
 296     }
 297     return $params;
 298   }
 299
 300   /**
 301    * Add progress bar to the import process. Calculates time remaining, status etc.
 302    *
 303    * @param $statusID
 304    *   status id of the import process saved in $config->uploadDir.
 305    * @param bool $startImport
 306    *   True when progress bar is to be initiated.
 307    * @param $startTimestamp
 308    *   Initial timstamp when the import was started.
 309    * @param $prevTimestamp
 310    *   Previous timestamp when this function was last called.
 311    * @param $totalRowCount
 312    *   Total number of rows in the import file.
 313    *
 314    * @return NULL|$currTimestamp
 315    */
 316   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 317     $config = CRM_Core_Config::singleton();
 318     $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
 319
 320     if ($startImport) {
 321       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 322       //do not force the browser to display the save dialog, CRM-7640
 323       $contents = json_encode([0, $status]);
 324       file_put_contents($statusFile, $contents);
 325     }
 326     else {
 327       $rowCount = isset($this->_rowCount) ? $this->_rowCount : $this->_lineCount;
 328       $currTimestamp = time();
 329       $totalTime = ($currTimestamp - $startTimestamp);
 330       $time = ($currTimestamp - $prevTimestamp);
 331       $recordsLeft = $totalRowCount - $rowCount;
 332       if ($recordsLeft < 0) {
 333         $recordsLeft = 0;
 334       }
 335       $estimatedTime = ($recordsLeft / 50) * $time;
 336       $estMinutes = floor($estimatedTime / 60);
 337       $timeFormatted = '';
 338       if ($estMinutes > 1) {
 339         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 340         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 341       }
 342       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 343       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 344       $statusMsg = ts('%1 of %2 records - %3 remaining',
 345         [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
 346       );
 347       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 348       $contents = json_encode([$processedPercent, $status]);
 349
 350       file_put_contents($statusFile, $contents);
 351       return $currTimestamp;
 352     }
 353   }
 354
 355   /**
 356    * @return array
 357    */
 358   public function getSelectValues() {
 359     $values = [];
 360     foreach ($this->_fields as $name => $field) {
 361       $values[$name] = $field->_title;
 362     }
 363     return $values;
 364   }
 365
 366   /**
 367    * @return array
 368    */
 369   public function getSelectTypes() {
 370     $values = [];
 371     foreach ($this->_fields as $name => $field) {
 372       if (isset($field->_hasLocationType)) {
 373         $values[$name] = $field->_hasLocationType;
 374       }
 375     }
 376     return $values;
 377   }
 378
 379   /**
 380    * @return array
 381    */
 382   public function getHeaderPatterns() {
 383     $values = [];
 384     foreach ($this->_fields as $name => $field) {
 385       if (isset($field->_headerPattern)) {
 386         $values[$name] = $field->_headerPattern;
 387       }
 388     }
 389     return $values;
 390   }
 391
 392   /**
 393    * @return array
 394    */
 395   public function getDataPatterns() {
 396     $values = [];
 397     foreach ($this->_fields as $name => $field) {
 398       $values[$name] = $field->_dataPattern;
 399     }
 400     return $values;
 401   }
 402
 403   /**
 404    * Remove single-quote enclosures from a value array (row).
 405    *
 406    * @param array $values
 407    * @param string $enclosure
 408    *
 409    * @return void
 410    */
 411   public static function encloseScrub(&$values, $enclosure = "'") {
 412     if (empty($values)) {
 413       return;
 414     }
 415
 416     foreach ($values as $k => $v) {
 417       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 418     }
 419   }
 420
 421   /**
 422    * Setter function.
 423    *
 424    * @param int $max
 425    *
 426    * @return void
 427    */
 428   public function setMaxLinesToProcess($max) {
 429     $this->_maxLinesToProcess = $max;
 430   }
 431
 432   /**
 433    * Determines the file extension based on error code.
 434    *
 435    * @var $type error code constant
 436    * @return string
 437    */
 438   public static function errorFileName($type) {
 439     $fileName = NULL;
 440     if (empty($type)) {
 441       return $fileName;
 442     }
 443
 444     $config = CRM_Core_Config::singleton();
 445     $fileName = $config->uploadDir . "sqlImport";
 446     switch ($type) {
 447       case self::ERROR:
 448         $fileName .= '.errors';
 449         break;
 450
 451       case self::CONFLICT:
 452         $fileName .= '.conflicts';
 453         break;
 454
 455       case self::DUPLICATE:
 456         $fileName .= '.duplicates';
 457         break;
 458
 459       case self::NO_MATCH:
 460         $fileName .= '.mismatch';
 461         break;
 462
 463       case self::UNPARSED_ADDRESS_WARNING:
 464         $fileName .= '.unparsedAddress';
 465         break;
 466     }
 467
 468     return $fileName;
 469   }
 470
 471   /**
 472    * Determines the file name based on error code.
 473    *
 474    * @var $type error code constant
 475    * @return string
 476    */
 477   public static function saveFileName($type) {
 478     $fileName = NULL;
 479     if (empty($type)) {
 480       return $fileName;
 481     }
 482     switch ($type) {
 483       case self::ERROR:
 484         $fileName = 'Import_Errors.csv';
 485         break;
 486
 487       case self::CONFLICT:
 488         $fileName = 'Import_Conflicts.csv';
 489         break;
 490
 491       case self::DUPLICATE:
 492         $fileName = 'Import_Duplicates.csv';
 493         break;
 494
 495       case self::NO_MATCH:
 496         $fileName = 'Import_Mismatch.csv';
 497         break;
 498
 499       case self::UNPARSED_ADDRESS_WARNING:
 500         $fileName = 'Import_Unparsed_Address.csv';
 501         break;
 502     }
 503
 504     return $fileName;
 505   }
 506
 507   /**
 508    * Check if contact is a duplicate .
 509    *
 510    * @param array $formatValues
 511    *
 512    * @return array
 513    */
 514   protected function checkContactDuplicate(&$formatValues) {
 515     //retrieve contact id using contact dedupe rule
 516     $formatValues['contact_type'] = $this->_contactType;
 517     $formatValues['version'] = 3;
 518     require_once 'CRM/Utils/DeprecatedUtils.php';
 519     $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
 520     return $error;
 521   }
 522
 523 }