CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2018                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2018
  32  */
  33 abstract class CRM_Import_Parser {
  34   /**
  35    * Settings
  36    */
  37   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  38
  39   /**
  40    * Return codes
  41    */
  42   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  43
  44   /**
  45    * Parser modes
  46    */
  47   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  48
  49   /**
  50    * Codes for duplicate record handling
  51    */
  52   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  53
  54   /**
  55    * Contact types
  56    */
  57   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  58
  59
  60   /**
  61    * Total number of non empty lines
  62    */
  63   protected $_totalCount;
  64
  65   /**
  66    * Running total number of valid lines
  67    */
  68   protected $_validCount;
  69
  70   /**
  71    * Running total number of invalid rows
  72    */
  73   protected $_invalidRowCount;
  74
  75   /**
  76    * Maximum number of non-empty/comment lines to process
  77    *
  78    * @var int
  79    */
  80   protected $_maxLinesToProcess;
  81
  82   /**
  83    * Array of error lines, bounded by MAX_ERROR
  84    */
  85   protected $_errors;
  86
  87   /**
  88    * Total number of conflict lines
  89    */
  90   protected $_conflictCount;
  91
  92   /**
  93    * Array of conflict lines
  94    */
  95   protected $_conflicts;
  96
  97   /**
  98    * Total number of duplicate (from database) lines
  99    */
 100   protected $_duplicateCount;
 101
 102   /**
 103    * Array of duplicate lines
 104    */
 105   protected $_duplicates;
 106
 107   /**
 108    * Running total number of warnings
 109    */
 110   protected $_warningCount;
 111
 112   /**
 113    * Maximum number of warnings to store
 114    */
 115   protected $_maxWarningCount = self::MAX_WARNINGS;
 116
 117   /**
 118    * Array of warning lines, bounded by MAX_WARNING
 119    */
 120   protected $_warnings;
 121
 122   /**
 123    * Array of all the fields that could potentially be part
 124    * of this import process
 125    * @var array
 126    */
 127   protected $_fields;
 128
 129   /**
 130    * Array of the fields that are actually part of the import process
 131    * the position in the array also dictates their position in the import
 132    * file
 133    * @var array
 134    */
 135   protected $_activeFields;
 136
 137   /**
 138    * Cache the count of active fields
 139    *
 140    * @var int
 141    */
 142   protected $_activeFieldCount;
 143
 144   /**
 145    * Cache of preview rows
 146    *
 147    * @var array
 148    */
 149   protected $_rows;
 150
 151   /**
 152    * Filename of error data
 153    *
 154    * @var string
 155    */
 156   protected $_errorFileName;
 157
 158   /**
 159    * Filename of conflict data
 160    *
 161    * @var string
 162    */
 163   protected $_conflictFileName;
 164
 165   /**
 166    * Filename of duplicate data
 167    *
 168    * @var string
 169    */
 170   protected $_duplicateFileName;
 171
 172   /**
 173    * Contact type
 174    *
 175    * @var int
 176    */
 177   public $_contactType;
 178   /**
 179    * Contact sub-type
 180    *
 181    * @var int
 182    */
 183   public $_contactSubType;
 184
 185   /**
 186    * Class constructor.
 187    */
 188   public function __construct() {
 189     $this->_maxLinesToProcess = 0;
 190   }
 191
 192   /**
 193    * Abstract function definitions.
 194    */
 195   abstract protected function init();
 196
 197   /**
 198    * @return mixed
 199    */
 200   abstract protected function fini();
 201
 202   /**
 203    * Map field.
 204    *
 205    * @param array $values
 206    *
 207    * @return mixed
 208    */
 209   abstract protected function mapField(&$values);
 210
 211   /**
 212    * Preview.
 213    *
 214    * @param array $values
 215    *
 216    * @return mixed
 217    */
 218   abstract protected function preview(&$values);
 219
 220   /**
 221    * @param $values
 222    *
 223    * @return mixed
 224    */
 225   abstract protected function summary(&$values);
 226
 227   /**
 228    * @param $onDuplicate
 229    * @param $values
 230    *
 231    * @return mixed
 232    */
 233   abstract protected function import($onDuplicate, &$values);
 234
 235   /**
 236    * Set and validate field values.
 237    *
 238    * @param array $elements
 239    *   array.
 240    * @param $erroneousField
 241    *   reference.
 242    *
 243    * @return int
 244    */
 245   public function setActiveFieldValues($elements, &$erroneousField) {
 246     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 247     for ($i = 0; $i < $maxCount; $i++) {
 248       $this->_activeFields[$i]->setValue($elements[$i]);
 249     }
 250
 251     // reset all the values that we did not have an equivalent import element
 252     for (; $i < $this->_activeFieldCount; $i++) {
 253       $this->_activeFields[$i]->resetValue();
 254     }
 255
 256     // now validate the fields and return false if error
 257     $valid = self::VALID;
 258     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 259       if (!$this->_activeFields[$i]->validate()) {
 260         // no need to do any more validation
 261         $erroneousField = $i;
 262         $valid = self::ERROR;
 263         break;
 264       }
 265     }
 266     return $valid;
 267   }
 268
 269   /**
 270    * Format the field values for input to the api.
 271    *
 272    * @return array
 273    *   (reference) associative array of name/value pairs
 274    */
 275   public function &getActiveFieldParams() {
 276     $params = array();
 277     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 278       if (isset($this->_activeFields[$i]->_value)
 279         && !isset($params[$this->_activeFields[$i]->_name])
 280         && !isset($this->_activeFields[$i]->_related)
 281       ) {
 282
 283         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 284       }
 285     }
 286     return $params;
 287   }
 288
 289   /**
 290    * Add progress bar to the import process. Calculates time remaining, status etc.
 291    *
 292    * @param $statusID
 293    *   status id of the import process saved in $config->uploadDir.
 294    * @param bool $startImport
 295    *   True when progress bar is to be initiated.
 296    * @param $startTimestamp
 297    *   Initial timstamp when the import was started.
 298    * @param $prevTimestamp
 299    *   Previous timestamp when this function was last called.
 300    * @param $totalRowCount
 301    *   Total number of rows in the import file.
 302    *
 303    * @return NULL|$currTimestamp
 304    */
 305   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 306     $config = CRM_Core_Config::singleton();
 307     $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
 308
 309     if ($startImport) {
 310       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 311       //do not force the browser to display the save dialog, CRM-7640
 312       $contents = json_encode(array(0, $status));
 313       file_put_contents($statusFile, $contents);
 314     }
 315     else {
 316       $rowCount = isset($this->_rowCount) ? $this->_rowCount : $this->_lineCount;
 317       $currTimestamp = time();
 318       $totalTime = ($currTimestamp - $startTimestamp);
 319       $time = ($currTimestamp - $prevTimestamp);
 320       $recordsLeft = $totalRowCount - $rowCount;
 321       if ($recordsLeft < 0) {
 322         $recordsLeft = 0;
 323       }
 324       $estimatedTime = ($recordsLeft / 50) * $time;
 325       $estMinutes = floor($estimatedTime / 60);
 326       $timeFormatted = '';
 327       if ($estMinutes > 1) {
 328         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 329         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 330       }
 331       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 332       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 333       $statusMsg = ts('%1 of %2 records - %3 remaining',
 334         array(1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted)
 335       );
 336       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 337       $contents = json_encode(array($processedPercent, $status));
 338
 339       file_put_contents($statusFile, $contents);
 340       return $currTimestamp;
 341     }
 342   }
 343
 344   /**
 345    * @return array
 346    */
 347   public function getSelectValues() {
 348     $values = array();
 349     foreach ($this->_fields as $name => $field) {
 350       $values[$name] = $field->_title;
 351     }
 352     return $values;
 353   }
 354
 355   /**
 356    * @return array
 357    */
 358   public function getSelectTypes() {
 359     $values = array();
 360     foreach ($this->_fields as $name => $field) {
 361       if (isset($field->_hasLocationType)) {
 362         $values[$name] = $field->_hasLocationType;
 363       }
 364     }
 365     return $values;
 366   }
 367
 368   /**
 369    * @return array
 370    */
 371   public function getHeaderPatterns() {
 372     $values = array();
 373     foreach ($this->_fields as $name => $field) {
 374       if (isset($field->_headerPattern)) {
 375         $values[$name] = $field->_headerPattern;
 376       }
 377     }
 378     return $values;
 379   }
 380
 381   /**
 382    * @return array
 383    */
 384   public function getDataPatterns() {
 385     $values = array();
 386     foreach ($this->_fields as $name => $field) {
 387       $values[$name] = $field->_dataPattern;
 388     }
 389     return $values;
 390   }
 391
 392   /**
 393    * Remove single-quote enclosures from a value array (row).
 394    *
 395    * @param array $values
 396    * @param string $enclosure
 397    *
 398    * @return void
 399    */
 400   public static function encloseScrub(&$values, $enclosure = "'") {
 401     if (empty($values)) {
 402       return;
 403     }
 404
 405     foreach ($values as $k => $v) {
 406       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 407     }
 408   }
 409
 410   /**
 411    * Setter function.
 412    *
 413    * @param int $max
 414    *
 415    * @return void
 416    */
 417   public function setMaxLinesToProcess($max) {
 418     $this->_maxLinesToProcess = $max;
 419   }
 420
 421   /**
 422    * Determines the file extension based on error code.
 423    *
 424    * @var $type error code constant
 425    * @return string
 426    */
 427   public static function errorFileName($type) {
 428     $fileName = NULL;
 429     if (empty($type)) {
 430       return $fileName;
 431     }
 432
 433     $config = CRM_Core_Config::singleton();
 434     $fileName = $config->uploadDir . "sqlImport";
 435     switch ($type) {
 436       case self::ERROR:
 437         $fileName .= '.errors';
 438         break;
 439
 440       case self::CONFLICT:
 441         $fileName .= '.conflicts';
 442         break;
 443
 444       case self::DUPLICATE:
 445         $fileName .= '.duplicates';
 446         break;
 447
 448       case self::NO_MATCH:
 449         $fileName .= '.mismatch';
 450         break;
 451
 452       case self::UNPARSED_ADDRESS_WARNING:
 453         $fileName .= '.unparsedAddress';
 454         break;
 455     }
 456
 457     return $fileName;
 458   }
 459
 460   /**
 461    * Determines the file name based on error code.
 462    *
 463    * @var $type error code constant
 464    * @return string
 465    */
 466   public static function saveFileName($type) {
 467     $fileName = NULL;
 468     if (empty($type)) {
 469       return $fileName;
 470     }
 471     switch ($type) {
 472       case self::ERROR:
 473         $fileName = 'Import_Errors.csv';
 474         break;
 475
 476       case self::CONFLICT:
 477         $fileName = 'Import_Conflicts.csv';
 478         break;
 479
 480       case self::DUPLICATE:
 481         $fileName = 'Import_Duplicates.csv';
 482         break;
 483
 484       case self::NO_MATCH:
 485         $fileName = 'Import_Mismatch.csv';
 486         break;
 487
 488       case self::UNPARSED_ADDRESS_WARNING:
 489         $fileName = 'Import_Unparsed_Address.csv';
 490         break;
 491     }
 492
 493     return $fileName;
 494   }
 495
 496   /**
 497    * Check if contact is a duplicate .
 498    *
 499    * @param array $formatValues
 500    *
 501    * @return array
 502    */
 503   protected function checkContactDuplicate(&$formatValues) {
 504     //retrieve contact id using contact dedupe rule
 505     $formatValues['contact_type'] = $this->_contactType;
 506     $formatValues['version'] = 3;
 507     require_once 'CRM/Utils/DeprecatedUtils.php';
 508     $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
 509     return $error;
 510   }
 511
 512 }