3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
17 abstract class CRM_Import_Parser
{
21 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
26 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
31 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
34 * Codes for duplicate record handling
36 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
41 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
45 * Total number of non empty lines
48 protected $_totalCount;
51 * Running total number of valid lines
54 protected $_validCount;
57 * Running total number of invalid rows
60 protected $_invalidRowCount;
63 * Maximum number of non-empty/comment lines to process
67 protected $_maxLinesToProcess;
70 * Array of error lines, bounded by MAX_ERROR
76 * Total number of conflict lines
79 protected $_conflictCount;
82 * Array of conflict lines
85 protected $_conflicts;
88 * Total number of duplicate (from database) lines
91 protected $_duplicateCount;
94 * Array of duplicate lines
97 protected $_duplicates;
100 * Running total number of warnings
103 protected $_warningCount;
106 * Maximum number of warnings to store
109 protected $_maxWarningCount = self
::MAX_WARNINGS
;
112 * Array of warning lines, bounded by MAX_WARNING
115 protected $_warnings;
118 * Array of all the fields that could potentially be part
119 * of this import process
125 * Metadata for all available fields, keyed by unique name.
127 * This is intended to supercede $_fields which uses a special sauce format which
128 * importableFieldsMetadata uses the standard getfields type format.
132 protected $importableFieldsMetadata = [];
135 * Get metadata for all importable fields in std getfields style format.
139 public function getImportableFieldsMetadata(): array {
140 return $this->importableFieldsMetadata
;
144 * Set metadata for all importable fields in std getfields style format.
145 * @param array $importableFieldsMetadata
147 public function setImportableFieldsMetadata(array $importableFieldsMetadata) {
148 $this->importableFieldsMetadata
= $importableFieldsMetadata;
152 * Array of the fields that are actually part of the import process
153 * the position in the array also dictates their position in the import
157 protected $_activeFields;
160 * Cache the count of active fields
164 protected $_activeFieldCount;
167 * Cache of preview rows
174 * Filename of error data
178 protected $_errorFileName;
181 * Filename of conflict data
185 protected $_conflictFileName;
188 * Filename of duplicate data
192 protected $_duplicateFileName;
199 public $_contactType;
205 public $_contactSubType;
210 public function __construct() {
211 $this->_maxLinesToProcess
= 0;
215 * Abstract function definitions.
217 abstract protected function init();
222 abstract protected function fini();
227 * @param array $values
231 abstract protected function mapField(&$values);
236 * @param array $values
240 abstract protected function preview(&$values);
247 abstract protected function summary(&$values);
250 * @param $onDuplicate
255 abstract protected function import($onDuplicate, &$values);
258 * Set and validate field values.
260 * @param array $elements
262 * @param $erroneousField
267 public function setActiveFieldValues($elements, &$erroneousField) {
268 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
269 for ($i = 0; $i < $maxCount; $i++
) {
270 $this->_activeFields
[$i]->setValue($elements[$i]);
273 // reset all the values that we did not have an equivalent import element
274 for (; $i < $this->_activeFieldCount
; $i++
) {
275 $this->_activeFields
[$i]->resetValue();
278 // now validate the fields and return false if error
279 $valid = self
::VALID
;
280 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
281 if (!$this->_activeFields
[$i]->validate()) {
282 // no need to do any more validation
283 $erroneousField = $i;
284 $valid = self
::ERROR
;
292 * Format the field values for input to the api.
295 * (reference) associative array of name/value pairs
297 public function &getActiveFieldParams() {
299 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
300 if (isset($this->_activeFields
[$i]->_value
)
301 && !isset($params[$this->_activeFields
[$i]->_name
])
302 && !isset($this->_activeFields
[$i]->_related
)
305 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
312 * Add progress bar to the import process. Calculates time remaining, status etc.
315 * status id of the import process saved in $config->uploadDir.
316 * @param bool $startImport
317 * True when progress bar is to be initiated.
318 * @param $startTimestamp
319 * Initial timstamp when the import was started.
320 * @param $prevTimestamp
321 * Previous timestamp when this function was last called.
322 * @param $totalRowCount
323 * Total number of rows in the import file.
325 * @return NULL|$currTimestamp
327 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
328 $config = CRM_Core_Config
::singleton();
329 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
332 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
333 //do not force the browser to display the save dialog, CRM-7640
334 $contents = json_encode([0, $status]);
335 file_put_contents($statusFile, $contents);
338 $rowCount = $this->_rowCount ??
$this->_lineCount
;
339 $currTimestamp = time();
340 $totalTime = ($currTimestamp - $startTimestamp);
341 $time = ($currTimestamp - $prevTimestamp);
342 $recordsLeft = $totalRowCount - $rowCount;
343 if ($recordsLeft < 0) {
346 $estimatedTime = ($recordsLeft / 50) * $time;
347 $estMinutes = floor($estimatedTime / 60);
349 if ($estMinutes > 1) {
350 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
351 $estimatedTime = $estimatedTime - ($estMinutes * 60);
353 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
354 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
355 $statusMsg = ts('%1 of %2 records - %3 remaining',
356 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
358 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
359 $contents = json_encode([$processedPercent, $status]);
361 file_put_contents($statusFile, $contents);
362 return $currTimestamp;
369 public function getSelectValues() {
371 foreach ($this->_fields
as $name => $field) {
372 $values[$name] = $field->_title
;
380 public function getSelectTypes() {
382 foreach ($this->_fields
as $name => $field) {
383 if (isset($field->_hasLocationType
)) {
384 $values[$name] = $field->_hasLocationType
;
393 public function getHeaderPatterns() {
395 foreach ($this->_fields
as $name => $field) {
396 if (isset($field->_headerPattern
)) {
397 $values[$name] = $field->_headerPattern
;
406 public function getDataPatterns() {
408 foreach ($this->_fields
as $name => $field) {
409 $values[$name] = $field->_dataPattern
;
415 * Remove single-quote enclosures from a value array (row).
417 * @param array $values
418 * @param string $enclosure
422 public static function encloseScrub(&$values, $enclosure = "'") {
423 if (empty($values)) {
427 foreach ($values as $k => $v) {
428 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
439 public function setMaxLinesToProcess($max) {
440 $this->_maxLinesToProcess
= $max;
444 * Determines the file extension based on error code.
446 * @var $type error code constant
449 public static function errorFileName($type) {
455 $config = CRM_Core_Config
::singleton();
456 $fileName = $config->uploadDir
. "sqlImport";
459 $fileName .= '.errors';
463 $fileName .= '.conflicts';
466 case self
::DUPLICATE
:
467 $fileName .= '.duplicates';
471 $fileName .= '.mismatch';
474 case self
::UNPARSED_ADDRESS_WARNING
:
475 $fileName .= '.unparsedAddress';
483 * Determines the file name based on error code.
485 * @var $type error code constant
488 public static function saveFileName($type) {
495 $fileName = 'Import_Errors.csv';
499 $fileName = 'Import_Conflicts.csv';
502 case self
::DUPLICATE
:
503 $fileName = 'Import_Duplicates.csv';
507 $fileName = 'Import_Mismatch.csv';
510 case self
::UNPARSED_ADDRESS_WARNING
:
511 $fileName = 'Import_Unparsed_Address.csv';
519 * Check if contact is a duplicate .
521 * @param array $formatValues
525 protected function checkContactDuplicate(&$formatValues) {
526 //retrieve contact id using contact dedupe rule
527 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->_contactType
;
528 $formatValues['version'] = 3;
529 require_once 'CRM/Utils/DeprecatedUtils.php';
530 $params = $formatValues;
531 static $cIndieFields = NULL;
532 static $defaultLocationId = NULL;
534 $contactType = $params['contact_type'];
535 if ($cIndieFields == NULL) {
536 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
537 $cIndieFields = $cTempIndieFields;
539 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
541 // set the value to default location id else set to 1
542 if (!$defaultLocationId = (int) $defaultLocation->id
) {
543 $defaultLocationId = 1;
547 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
549 $contactFormatted = [];
550 foreach ($params as $key => $field) {
551 if ($field == NULL ||
$field === '') {
554 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
555 // instead of soft credit contact.
556 if (is_array($field) && $key != "soft_credit") {
557 foreach ($field as $value) {
559 if (is_array($value)) {
560 foreach ($value as $name => $testForEmpty) {
561 if ($name !== 'phone_type' &&
562 ($testForEmpty === '' ||
$testForEmpty == NULL)
573 _civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
579 $value = [$key => $field];
581 // check if location related field, then we need to add primary location type
582 if (in_array($key, $locationFields)) {
583 $value['location_type_id'] = $defaultLocationId;
585 elseif (array_key_exists($key, $cIndieFields)) {
586 $value['contact_type'] = $contactType;
589 _civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
592 $contactFormatted['contact_type'] = $contactType;
594 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
598 * Parse a field which could be represented by a label or name value rather than the DB value.
600 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
602 * but if not available then see if we have a label that can be converted to a name.
604 * @param string|int|null $submittedValue
605 * @param array $fieldSpec
606 * Metadata for the field
610 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
611 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
612 if (!isset($fieldSpec['bao'])) {
613 return $submittedValue;
615 /* @var \CRM_Core_DAO $bao */
616 $bao = $fieldSpec['bao'];
617 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
618 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
619 if (isset($nameOptions[$submittedValue])) {
620 return $submittedValue;
622 if (in_array($submittedValue, $nameOptions)) {
623 return array_search($submittedValue, $nameOptions, TRUE);
626 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
627 if (isset($labelOptions[$submittedValue])) {
628 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
634 * This is code extracted from 4 places where this exact snippet was being duplicated.
636 * FIXME: Extracting this was a first step, but there's also
637 * 1. Inconsistency in the way other select options are handled.
638 * Contribution adds handling for Select/Radio/Autocomplete
639 * Participant/Activity only handles Select/Radio and misses Autocomplete
640 * Membership is missing all of it
641 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
643 * @param $customFieldID
648 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
649 $mulValues = explode(',', $value);
650 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
652 foreach ($mulValues as $v1) {
653 foreach ($customOption as $customValueID => $customLabel) {
654 $customValue = $customLabel['value'];
655 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
656 (strtolower(trim($customValue)) == strtolower(trim($v1)))
658 if ($fieldType == 'CheckBox') {
659 $values[$customValue] = 1;
662 $values[] = $customValue;
671 * Get the ids of any contacts that match according to the rule.
673 * @param array $formatted
677 protected function getIdsOfMatchingContacts(array $formatted):array {
678 // the call to the deprecated function seems to add no value other that to do an additional
679 // check for the contact_id & type.
680 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
681 if (!CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
684 if (is_array($error['error_message']['params'][0])) {
685 return $error['error_message']['params'][0];
688 return explode(',', $error['error_message']['params'][0]);