3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2019
33 abstract class CRM_Import_Parser
{
37 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
42 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
47 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
50 * Codes for duplicate record handling
52 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
57 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
61 * Total number of non empty lines
64 protected $_totalCount;
67 * Running total number of valid lines
70 protected $_validCount;
73 * Running total number of invalid rows
76 protected $_invalidRowCount;
79 * Maximum number of non-empty/comment lines to process
83 protected $_maxLinesToProcess;
86 * Array of error lines, bounded by MAX_ERROR
92 * Total number of conflict lines
95 protected $_conflictCount;
98 * Array of conflict lines
101 protected $_conflicts;
104 * Total number of duplicate (from database) lines
107 protected $_duplicateCount;
110 * Array of duplicate lines
113 protected $_duplicates;
116 * Running total number of warnings
119 protected $_warningCount;
122 * Maximum number of warnings to store
125 protected $_maxWarningCount = self
::MAX_WARNINGS
;
128 * Array of warning lines, bounded by MAX_WARNING
131 protected $_warnings;
134 * Array of all the fields that could potentially be part
135 * of this import process
141 * Metadata for all available fields, keyed by unique name.
143 * This is intended to supercede $_fields which uses a special sauce format which
144 * importableFieldsMetadata uses the standard getfields type format.
148 protected $importableFieldsMetadata = [];
151 * Get metadata for all importable fields in std getfields style format.
155 public function getImportableFieldsMetadata(): array {
156 return $this->importableFieldsMetadata
;
160 * Set metadata for all importable fields in std getfields style format.
161 * @param array $importableFieldsMetadata
163 public function setImportableFieldsMetadata(array $importableFieldsMetadata) {
164 $this->importableFieldsMetadata
= $importableFieldsMetadata;
168 * Array of the fields that are actually part of the import process
169 * the position in the array also dictates their position in the import
173 protected $_activeFields;
176 * Cache the count of active fields
180 protected $_activeFieldCount;
183 * Cache of preview rows
190 * Filename of error data
194 protected $_errorFileName;
197 * Filename of conflict data
201 protected $_conflictFileName;
204 * Filename of duplicate data
208 protected $_duplicateFileName;
215 public $_contactType;
221 public $_contactSubType;
226 public function __construct() {
227 $this->_maxLinesToProcess
= 0;
231 * Abstract function definitions.
233 abstract protected function init();
238 abstract protected function fini();
243 * @param array $values
247 abstract protected function mapField(&$values);
252 * @param array $values
256 abstract protected function preview(&$values);
263 abstract protected function summary(&$values);
266 * @param $onDuplicate
271 abstract protected function import($onDuplicate, &$values);
274 * Set and validate field values.
276 * @param array $elements
278 * @param $erroneousField
283 public function setActiveFieldValues($elements, &$erroneousField) {
284 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
285 for ($i = 0; $i < $maxCount; $i++
) {
286 $this->_activeFields
[$i]->setValue($elements[$i]);
289 // reset all the values that we did not have an equivalent import element
290 for (; $i < $this->_activeFieldCount
; $i++
) {
291 $this->_activeFields
[$i]->resetValue();
294 // now validate the fields and return false if error
295 $valid = self
::VALID
;
296 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
297 if (!$this->_activeFields
[$i]->validate()) {
298 // no need to do any more validation
299 $erroneousField = $i;
300 $valid = self
::ERROR
;
308 * Format the field values for input to the api.
311 * (reference) associative array of name/value pairs
313 public function &getActiveFieldParams() {
315 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
316 if (isset($this->_activeFields
[$i]->_value
)
317 && !isset($params[$this->_activeFields
[$i]->_name
])
318 && !isset($this->_activeFields
[$i]->_related
)
321 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
328 * Add progress bar to the import process. Calculates time remaining, status etc.
331 * status id of the import process saved in $config->uploadDir.
332 * @param bool $startImport
333 * True when progress bar is to be initiated.
334 * @param $startTimestamp
335 * Initial timstamp when the import was started.
336 * @param $prevTimestamp
337 * Previous timestamp when this function was last called.
338 * @param $totalRowCount
339 * Total number of rows in the import file.
341 * @return NULL|$currTimestamp
343 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
344 $config = CRM_Core_Config
::singleton();
345 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
348 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
349 //do not force the browser to display the save dialog, CRM-7640
350 $contents = json_encode([0, $status]);
351 file_put_contents($statusFile, $contents);
354 $rowCount = isset($this->_rowCount
) ?
$this->_rowCount
: $this->_lineCount
;
355 $currTimestamp = time();
356 $totalTime = ($currTimestamp - $startTimestamp);
357 $time = ($currTimestamp - $prevTimestamp);
358 $recordsLeft = $totalRowCount - $rowCount;
359 if ($recordsLeft < 0) {
362 $estimatedTime = ($recordsLeft / 50) * $time;
363 $estMinutes = floor($estimatedTime / 60);
365 if ($estMinutes > 1) {
366 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
367 $estimatedTime = $estimatedTime - ($estMinutes * 60);
369 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
370 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
371 $statusMsg = ts('%1 of %2 records - %3 remaining',
372 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
374 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
375 $contents = json_encode([$processedPercent, $status]);
377 file_put_contents($statusFile, $contents);
378 return $currTimestamp;
385 public function getSelectValues() {
387 foreach ($this->_fields
as $name => $field) {
388 $values[$name] = $field->_title
;
396 public function getSelectTypes() {
398 foreach ($this->_fields
as $name => $field) {
399 if (isset($field->_hasLocationType
)) {
400 $values[$name] = $field->_hasLocationType
;
409 public function getHeaderPatterns() {
411 foreach ($this->_fields
as $name => $field) {
412 if (isset($field->_headerPattern
)) {
413 $values[$name] = $field->_headerPattern
;
422 public function getDataPatterns() {
424 foreach ($this->_fields
as $name => $field) {
425 $values[$name] = $field->_dataPattern
;
431 * Remove single-quote enclosures from a value array (row).
433 * @param array $values
434 * @param string $enclosure
438 public static function encloseScrub(&$values, $enclosure = "'") {
439 if (empty($values)) {
443 foreach ($values as $k => $v) {
444 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
455 public function setMaxLinesToProcess($max) {
456 $this->_maxLinesToProcess
= $max;
460 * Determines the file extension based on error code.
462 * @var $type error code constant
465 public static function errorFileName($type) {
471 $config = CRM_Core_Config
::singleton();
472 $fileName = $config->uploadDir
. "sqlImport";
475 $fileName .= '.errors';
479 $fileName .= '.conflicts';
482 case self
::DUPLICATE
:
483 $fileName .= '.duplicates';
487 $fileName .= '.mismatch';
490 case self
::UNPARSED_ADDRESS_WARNING
:
491 $fileName .= '.unparsedAddress';
499 * Determines the file name based on error code.
501 * @var $type error code constant
504 public static function saveFileName($type) {
511 $fileName = 'Import_Errors.csv';
515 $fileName = 'Import_Conflicts.csv';
518 case self
::DUPLICATE
:
519 $fileName = 'Import_Duplicates.csv';
523 $fileName = 'Import_Mismatch.csv';
526 case self
::UNPARSED_ADDRESS_WARNING
:
527 $fileName = 'Import_Unparsed_Address.csv';
535 * Check if contact is a duplicate .
537 * @param array $formatValues
541 protected function checkContactDuplicate(&$formatValues) {
542 //retrieve contact id using contact dedupe rule
543 $formatValues['contact_type'] = $this->_contactType
;
544 $formatValues['version'] = 3;
545 require_once 'CRM/Utils/DeprecatedUtils.php';
546 $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
551 * Parse a field which could be represented by a label or name value rather than the DB value.
553 * We will try to match name first but if not available then see if we have a label that can be converted to a name.
555 * @param string|int|null $submittedValue
556 * @param array $fieldSpec
557 * Metadata for the field
561 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
562 /* @var \CRM_Core_DAO $bao */
563 $bao = $fieldSpec['bao'];
564 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
565 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
566 if (!isset($nameOptions[$submittedValue])) {
567 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
568 if (isset($labelOptions[$submittedValue])) {
569 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);