3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2019
33 abstract class CRM_Import_Parser
{
37 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
42 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
47 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
50 * Codes for duplicate record handling
52 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
57 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
61 * Total number of non empty lines
64 protected $_totalCount;
67 * Running total number of valid lines
70 protected $_validCount;
73 * Running total number of invalid rows
76 protected $_invalidRowCount;
79 * Maximum number of non-empty/comment lines to process
83 protected $_maxLinesToProcess;
86 * Array of error lines, bounded by MAX_ERROR
92 * Total number of conflict lines
95 protected $_conflictCount;
98 * Array of conflict lines
101 protected $_conflicts;
104 * Total number of duplicate (from database) lines
107 protected $_duplicateCount;
110 * Array of duplicate lines
113 protected $_duplicates;
116 * Running total number of warnings
119 protected $_warningCount;
122 * Maximum number of warnings to store
125 protected $_maxWarningCount = self
::MAX_WARNINGS
;
128 * Array of warning lines, bounded by MAX_WARNING
131 protected $_warnings;
134 * Array of all the fields that could potentially be part
135 * of this import process
141 * Metadata for all available fields, keyed by unique name.
143 * This is intended to supercede $_fields which uses a special sauce format which
144 * importableFieldsMetadata uses the standard getfields type format.
148 protected $importableFieldsMetadata = [];
151 * Get metadata for all importable fields in std getfields style format.
155 public function getImportableFieldsMetadata(): array {
156 return $this->importableFieldsMetadata
;
160 * Set metadata for all importable fields in std getfields style format.
161 * @param array $importableFieldsMetadata
163 public function setImportableFieldsMetadata(array $importableFieldsMetadata) {
164 $this->importableFieldsMetadata
= $importableFieldsMetadata;
168 * Array of the fields that are actually part of the import process
169 * the position in the array also dictates their position in the import
173 protected $_activeFields;
176 * Cache the count of active fields
180 protected $_activeFieldCount;
183 * Cache of preview rows
190 * Filename of error data
194 protected $_errorFileName;
197 * Filename of conflict data
201 protected $_conflictFileName;
204 * Filename of duplicate data
208 protected $_duplicateFileName;
215 public $_contactType;
221 public $_contactSubType;
226 public function __construct() {
227 $this->_maxLinesToProcess
= 0;
231 * Abstract function definitions.
233 abstract protected function init();
238 abstract protected function fini();
243 * @param array $values
247 abstract protected function mapField(&$values);
252 * @param array $values
256 abstract protected function preview(&$values);
263 abstract protected function summary(&$values);
266 * @param $onDuplicate
271 abstract protected function import($onDuplicate, &$values);
274 * Set and validate field values.
276 * @param array $elements
278 * @param $erroneousField
283 public function setActiveFieldValues($elements, &$erroneousField) {
284 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
285 for ($i = 0; $i < $maxCount; $i++
) {
286 $this->_activeFields
[$i]->setValue($elements[$i]);
289 // reset all the values that we did not have an equivalent import element
290 for (; $i < $this->_activeFieldCount
; $i++
) {
291 $this->_activeFields
[$i]->resetValue();
294 // now validate the fields and return false if error
295 $valid = self
::VALID
;
296 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
297 if (!$this->_activeFields
[$i]->validate()) {
298 // no need to do any more validation
299 $erroneousField = $i;
300 $valid = self
::ERROR
;
308 * Format the field values for input to the api.
311 * (reference) associative array of name/value pairs
313 public function &getActiveFieldParams() {
315 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
316 if (isset($this->_activeFields
[$i]->_value
)
317 && !isset($params[$this->_activeFields
[$i]->_name
])
318 && !isset($this->_activeFields
[$i]->_related
)
321 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
328 * Add progress bar to the import process. Calculates time remaining, status etc.
331 * status id of the import process saved in $config->uploadDir.
332 * @param bool $startImport
333 * True when progress bar is to be initiated.
334 * @param $startTimestamp
335 * Initial timstamp when the import was started.
336 * @param $prevTimestamp
337 * Previous timestamp when this function was last called.
338 * @param $totalRowCount
339 * Total number of rows in the import file.
341 * @return NULL|$currTimestamp
343 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
344 $config = CRM_Core_Config
::singleton();
345 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
348 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
349 //do not force the browser to display the save dialog, CRM-7640
350 $contents = json_encode([0, $status]);
351 file_put_contents($statusFile, $contents);
354 $rowCount = isset($this->_rowCount
) ?
$this->_rowCount
: $this->_lineCount
;
355 $currTimestamp = time();
356 $totalTime = ($currTimestamp - $startTimestamp);
357 $time = ($currTimestamp - $prevTimestamp);
358 $recordsLeft = $totalRowCount - $rowCount;
359 if ($recordsLeft < 0) {
362 $estimatedTime = ($recordsLeft / 50) * $time;
363 $estMinutes = floor($estimatedTime / 60);
365 if ($estMinutes > 1) {
366 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
367 $estimatedTime = $estimatedTime - ($estMinutes * 60);
369 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
370 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
371 $statusMsg = ts('%1 of %2 records - %3 remaining',
372 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
374 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
375 $contents = json_encode([$processedPercent, $status]);
377 file_put_contents($statusFile, $contents);
378 return $currTimestamp;
385 public function getSelectValues() {
387 foreach ($this->_fields
as $name => $field) {
388 $values[$name] = $field->_title
;
396 public function getSelectTypes() {
398 foreach ($this->_fields
as $name => $field) {
399 if (isset($field->_hasLocationType
)) {
400 $values[$name] = $field->_hasLocationType
;
409 public function getHeaderPatterns() {
411 foreach ($this->_fields
as $name => $field) {
412 if (isset($field->_headerPattern
)) {
413 $values[$name] = $field->_headerPattern
;
422 public function getDataPatterns() {
424 foreach ($this->_fields
as $name => $field) {
425 $values[$name] = $field->_dataPattern
;
431 * Remove single-quote enclosures from a value array (row).
433 * @param array $values
434 * @param string $enclosure
438 public static function encloseScrub(&$values, $enclosure = "'") {
439 if (empty($values)) {
443 foreach ($values as $k => $v) {
444 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
455 public function setMaxLinesToProcess($max) {
456 $this->_maxLinesToProcess
= $max;
460 * Determines the file extension based on error code.
462 * @var $type error code constant
465 public static function errorFileName($type) {
471 $config = CRM_Core_Config
::singleton();
472 $fileName = $config->uploadDir
. "sqlImport";
475 $fileName .= '.errors';
479 $fileName .= '.conflicts';
482 case self
::DUPLICATE
:
483 $fileName .= '.duplicates';
487 $fileName .= '.mismatch';
490 case self
::UNPARSED_ADDRESS_WARNING
:
491 $fileName .= '.unparsedAddress';
499 * Determines the file name based on error code.
501 * @var $type error code constant
504 public static function saveFileName($type) {
511 $fileName = 'Import_Errors.csv';
515 $fileName = 'Import_Conflicts.csv';
518 case self
::DUPLICATE
:
519 $fileName = 'Import_Duplicates.csv';
523 $fileName = 'Import_Mismatch.csv';
526 case self
::UNPARSED_ADDRESS_WARNING
:
527 $fileName = 'Import_Unparsed_Address.csv';
535 * Check if contact is a duplicate .
537 * @param array $formatValues
541 protected function checkContactDuplicate(&$formatValues) {
542 //retrieve contact id using contact dedupe rule
543 $formatValues['contact_type'] = $this->_contactType
;
544 $formatValues['version'] = 3;
545 require_once 'CRM/Utils/DeprecatedUtils.php';
546 $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
551 * Parse a field which could be represented by a label or name value rather than the DB value.
553 * We will try to match name first but if not available then see if we have a label that can be converted to a name.
555 * @param string|int|null $submittedValue
556 * @param array $fieldSpec
557 * Metadata for the field
561 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
562 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
563 if (!isset($fieldSpec['bao'])) {
564 return $submittedValue;
566 /* @var \CRM_Core_DAO $bao */
567 $bao = $fieldSpec['bao'];
568 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
569 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
570 if (!isset($nameOptions[$submittedValue])) {
571 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
572 if (isset($labelOptions[$submittedValue])) {
573 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);