CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2017                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2017
  32  */
  33 abstract class CRM_Import_Parser {
  34   /**
  35    * Settings
  36    */
  37   const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30;
  38
  39   /**
  40    * Return codes
  41    */
  42   const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  43
  44   /**
  45    * Parser modes
  46    */
  47   const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  48
  49   /**
  50    * Codes for duplicate record handling
  51    */
  52   const DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  53
  54   /**
  55    * Contact types
  56    */
  57   const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  58
  59
  60   /**
  61    * Total number of non empty lines
  62    */
  63   protected $_totalCount;
  64
  65   /**
  66    * Running total number of valid lines
  67    */
  68   protected $_validCount;
  69
  70   /**
  71    * Running total number of invalid rows
  72    */
  73   protected $_invalidRowCount;
  74
  75   /**
  76    * Maximum number of non-empty/comment lines to process
  77    *
  78    * @var int
  79    */
  80   protected $_maxLinesToProcess;
  81
  82   /**
  83    * Array of error lines, bounded by MAX_ERROR
  84    */
  85   protected $_errors;
  86
  87   /**
  88    * Total number of conflict lines
  89    */
  90   protected $_conflictCount;
  91
  92   /**
  93    * Array of conflict lines
  94    */
  95   protected $_conflicts;
  96
  97   /**
  98    * Total number of duplicate (from database) lines
  99    */
 100   protected $_duplicateCount;
 101
 102   /**
 103    * Array of duplicate lines
 104    */
 105   protected $_duplicates;
 106
 107   /**
 108    * Running total number of warnings
 109    */
 110   protected $_warningCount;
 111
 112   /**
 113    * Maximum number of warnings to store
 114    */
 115   protected $_maxWarningCount = self::MAX_WARNINGS;
 116
 117   /**
 118    * Array of warning lines, bounded by MAX_WARNING
 119    */
 120   protected $_warnings;
 121
 122   /**
 123    * Array of all the fields that could potentially be part
 124    * of this import process
 125    * @var array
 126    */
 127   protected $_fields;
 128
 129   /**
 130    * Array of the fields that are actually part of the import process
 131    * the position in the array also dictates their position in the import
 132    * file
 133    * @var array
 134    */
 135   protected $_activeFields;
 136
 137   /**
 138    * Cache the count of active fields
 139    *
 140    * @var int
 141    */
 142   protected $_activeFieldCount;
 143
 144   /**
 145    * Cache of preview rows
 146    *
 147    * @var array
 148    */
 149   protected $_rows;
 150
 151   /**
 152    * Filename of error data
 153    *
 154    * @var string
 155    */
 156   protected $_errorFileName;
 157
 158   /**
 159    * Filename of conflict data
 160    *
 161    * @var string
 162    */
 163   protected $_conflictFileName;
 164
 165   /**
 166    * Filename of duplicate data
 167    *
 168    * @var string
 169    */
 170   protected $_duplicateFileName;
 171
 172   /**
 173    * Contact type
 174    *
 175    * @var int
 176    */
 177   public $_contactType;
 178   /**
 179    * Contact sub-type
 180    *
 181    * @var int
 182    */
 183   public $_contactSubType;
 184
 185   /**
 186    * Class constructor.
 187    */
 188   public function __construct() {
 189     $this->_maxLinesToProcess = 0;
 190   }
 191
 192   /**
 193    * Abstract function definitions.
 194    */
 195   abstract protected function init();
 196
 197   /**
 198    * @return mixed
 199    */
 200   abstract protected function fini();
 201
 202   /**
 203    * Map field.
 204    *
 205    * @param array $values
 206    *
 207    * @return mixed
 208    */
 209   abstract protected function mapField(&$values);
 210
 211   /**
 212    * Preview.
 213    *
 214    * @param array $values
 215    *
 216    * @return mixed
 217    */
 218   abstract protected function preview(&$values);
 219
 220   /**
 221    * @param $values
 222    *
 223    * @return mixed
 224    */
 225   abstract protected function summary(&$values);
 226
 227   /**
 228    * @param $onDuplicate
 229    * @param $values
 230    *
 231    * @return mixed
 232    */
 233   abstract protected function import($onDuplicate, &$values);
 234
 235   /**
 236    * Set and validate field values.
 237    *
 238    * @param array $elements
 239    *   array.
 240    * @param $erroneousField
 241    *   reference.
 242    *
 243    * @return int
 244    */
 245   public function setActiveFieldValues($elements, &$erroneousField) {
 246     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 247     for ($i = 0; $i < $maxCount; $i++) {
 248       $this->_activeFields[$i]->setValue($elements[$i]);
 249     }
 250
 251     // reset all the values that we did not have an equivalent import element
 252     for (; $i < $this->_activeFieldCount; $i++) {
 253       $this->_activeFields[$i]->resetValue();
 254     }
 255
 256     // now validate the fields and return false if error
 257     $valid = self::VALID;
 258     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 259       if (!$this->_activeFields[$i]->validate()) {
 260         // no need to do any more validation
 261         $erroneousField = $i;
 262         $valid = self::ERROR;
 263         break;
 264       }
 265     }
 266     return $valid;
 267   }
 268
 269   /**
 270    * Format the field values for input to the api.
 271    *
 272    * @return array
 273    *   (reference) associative array of name/value pairs
 274    */
 275   public function &getActiveFieldParams() {
 276     $params = array();
 277     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 278       if (isset($this->_activeFields[$i]->_value)
 279         && !isset($params[$this->_activeFields[$i]->_name])
 280         && !isset($this->_activeFields[$i]->_related)
 281       ) {
 282
 283         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 284       }
 285     }
 286     return $params;
 287   }
 288
 289   /**
 290    * @param $statusID
 291    * @param bool $startImport
 292    *   True when progress bar is to be initiated.
 293    * @param $startTimestamp
 294    *   Initial timstamp when the import was started.
 295    * @param $prevTimestamp
 296    *   Previous timestamp when this function was last called.
 297    * @param $totalRowCount
 298    *   Total number of rows in the import file.
 299    *
 300    * @return NULL|$currTimestamp
 301    */
 302   public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
 303     $config = CRM_Core_Config::singleton();
 304     $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
 305
 306     if ($startImport) {
 307       $status = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 308       //do not force the browser to display the save dialog, CRM-7640
 309       $contents = json_encode(array(0, $status));
 310       file_put_contents($statusFile, $contents);
 311     }
 312     else {
 313       $rowCount = isset($this->_rowCount) ? $this->_rowCount : $this->_lineCount;
 314       $currTimestamp = time();
 315       $totalTime = ($currTimestamp - $startTimestamp);
 316       $time = ($currTimestamp - $prevTimestamp);
 317       $recordsLeft = $totalRowCount - $rowCount;
 318       if ($recordsLeft < 0) {
 319         $recordsLeft = 0;
 320       }
 321       $estimatedTime = ($recordsLeft / 50) * $time;
 322       $estMinutes = floor($estimatedTime / 60);
 323       $timeFormatted = '';
 324       if ($estMinutes > 1) {
 325         $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 326         $estimatedTime = $estimatedTime - ($estMinutes * 60);
 327       }
 328       $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 329       $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
 330       $statusMsg = ts('%1 of %2 records - %3 remaining',
 331         array(1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted)
 332       );
 333       $status = "<div class=\"description\">&nbsp; <strong>{$statusMsg}</strong></div>";
 334       $contents = json_encode(array($processedPercent, $status));
 335
 336       file_put_contents($statusFile, $contents);
 337       return $currTimestamp;
 338     }
 339   }
 340
 341   /**
 342    * @return array
 343    */
 344   public function getSelectValues() {
 345     $values = array();
 346     foreach ($this->_fields as $name => $field) {
 347       $values[$name] = $field->_title;
 348     }
 349     return $values;
 350   }
 351
 352   /**
 353    * @return array
 354    */
 355   public function getSelectTypes() {
 356     $values = array();
 357     foreach ($this->_fields as $name => $field) {
 358       if (isset($field->_hasLocationType)) {
 359         $values[$name] = $field->_hasLocationType;
 360       }
 361     }
 362     return $values;
 363   }
 364
 365   /**
 366    * @return array
 367    */
 368   public function getHeaderPatterns() {
 369     $values = array();
 370     foreach ($this->_fields as $name => $field) {
 371       if (isset($field->_headerPattern)) {
 372         $values[$name] = $field->_headerPattern;
 373       }
 374     }
 375     return $values;
 376   }
 377
 378   /**
 379    * @return array
 380    */
 381   public function getDataPatterns() {
 382     $values = array();
 383     foreach ($this->_fields as $name => $field) {
 384       $values[$name] = $field->_dataPattern;
 385     }
 386     return $values;
 387   }
 388
 389   /**
 390    * Remove single-quote enclosures from a value array (row).
 391    *
 392    * @param array $values
 393    * @param string $enclosure
 394    *
 395    * @return void
 396    */
 397   public static function encloseScrub(&$values, $enclosure = "'") {
 398     if (empty($values)) {
 399       return;
 400     }
 401
 402     foreach ($values as $k => $v) {
 403       $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
 404     }
 405   }
 406
 407   /**
 408    * Setter function.
 409    *
 410    * @param int $max
 411    *
 412    * @return void
 413    */
 414   public function setMaxLinesToProcess($max) {
 415     $this->_maxLinesToProcess = $max;
 416   }
 417
 418   /**
 419    * Determines the file extension based on error code.
 420    *
 421    * @var $type error code constant
 422    * @return string
 423    */
 424   public static function errorFileName($type) {
 425     $fileName = NULL;
 426     if (empty($type)) {
 427       return $fileName;
 428     }
 429
 430     $config = CRM_Core_Config::singleton();
 431     $fileName = $config->uploadDir . "sqlImport";
 432     switch ($type) {
 433       case self::ERROR:
 434         $fileName .= '.errors';
 435         break;
 436
 437       case self::CONFLICT:
 438         $fileName .= '.conflicts';
 439         break;
 440
 441       case self::DUPLICATE:
 442         $fileName .= '.duplicates';
 443         break;
 444
 445       case self::NO_MATCH:
 446         $fileName .= '.mismatch';
 447         break;
 448
 449       case self::UNPARSED_ADDRESS_WARNING:
 450         $fileName .= '.unparsedAddress';
 451         break;
 452     }
 453
 454     return $fileName;
 455   }
 456
 457   /**
 458    * Determines the file name based on error code.
 459    *
 460    * @var $type error code constant
 461    * @return string
 462    */
 463   public static function saveFileName($type) {
 464     $fileName = NULL;
 465     if (empty($type)) {
 466       return $fileName;
 467     }
 468     switch ($type) {
 469       case self::ERROR:
 470         $fileName = 'Import_Errors.csv';
 471         break;
 472
 473       case self::CONFLICT:
 474         $fileName = 'Import_Conflicts.csv';
 475         break;
 476
 477       case self::DUPLICATE:
 478         $fileName = 'Import_Duplicates.csv';
 479         break;
 480
 481       case self::NO_MATCH:
 482         $fileName = 'Import_Mismatch.csv';
 483         break;
 484
 485       case self::UNPARSED_ADDRESS_WARNING:
 486         $fileName = 'Import_Unparsed_Address.csv';
 487         break;
 488     }
 489
 490     return $fileName;
 491   }
 492
 493   /**
 494    * Check if contact is a duplicate .
 495    *
 496    * @param array $formatValues
 497    *
 498    * @return array
 499    */
 500   protected function checkContactDuplicate(&$formatValues) {
 501     //retrieve contact id using contact dedupe rule
 502     $formatValues['contact_type'] = $this->_contactType;
 503     $formatValues['version'] = 3;
 504     require_once 'CRM/Utils/DeprecatedUtils.php';
 505     $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
 506     return $error;
 507   }
 508
 509 }