3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
17 abstract class CRM_Import_Parser
{
21 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
26 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
31 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
34 * Codes for duplicate record handling
36 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
41 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
45 * Total number of non empty lines
48 protected $_totalCount;
51 * Running total number of valid lines
54 protected $_validCount;
57 * Running total number of invalid rows
60 protected $_invalidRowCount;
63 * Maximum number of non-empty/comment lines to process
67 protected $_maxLinesToProcess;
70 * Array of error lines, bounded by MAX_ERROR
76 * Total number of conflict lines
79 protected $_conflictCount;
82 * Array of conflict lines
85 protected $_conflicts;
88 * Total number of duplicate (from database) lines
91 protected $_duplicateCount;
94 * Array of duplicate lines
97 protected $_duplicates;
100 * Running total number of warnings
103 protected $_warningCount;
106 * Maximum number of warnings to store
109 protected $_maxWarningCount = self
::MAX_WARNINGS
;
112 * Array of warning lines, bounded by MAX_WARNING
115 protected $_warnings;
118 * Array of all the fields that could potentially be part
119 * of this import process
125 * Metadata for all available fields, keyed by unique name.
127 * This is intended to supercede $_fields which uses a special sauce format which
128 * importableFieldsMetadata uses the standard getfields type format.
132 protected $importableFieldsMetadata = [];
135 * Get metadata for all importable fields in std getfields style format.
139 public function getImportableFieldsMetadata(): array {
140 return $this->importableFieldsMetadata
;
144 * Set metadata for all importable fields in std getfields style format.
146 * @param array $importableFieldsMetadata
148 public function setImportableFieldsMetadata(array $importableFieldsMetadata): void
{
149 $this->importableFieldsMetadata
= $importableFieldsMetadata;
153 * Array of the fields that are actually part of the import process
154 * the position in the array also dictates their position in the import
158 protected $_activeFields;
161 * Cache the count of active fields
165 protected $_activeFieldCount;
168 * Cache of preview rows
175 * Filename of error data
179 protected $_errorFileName;
182 * Filename of conflict data
186 protected $_conflictFileName;
189 * Filename of duplicate data
193 protected $_duplicateFileName;
200 public $_contactType;
206 public $_contactSubType;
211 public function __construct() {
212 $this->_maxLinesToProcess
= 0;
216 * Abstract function definitions.
218 abstract protected function init();
223 abstract protected function fini();
228 * @param array $values
232 abstract protected function mapField(&$values);
237 * @param array $values
241 abstract protected function preview(&$values);
248 abstract protected function summary(&$values);
251 * @param $onDuplicate
256 abstract protected function import($onDuplicate, &$values);
259 * Set and validate field values.
261 * @param array $elements
263 * @param $erroneousField
268 public function setActiveFieldValues($elements, &$erroneousField = NULL) {
269 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
270 for ($i = 0; $i < $maxCount; $i++
) {
271 $this->_activeFields
[$i]->setValue($elements[$i]);
274 // reset all the values that we did not have an equivalent import element
275 for (; $i < $this->_activeFieldCount
; $i++
) {
276 $this->_activeFields
[$i]->resetValue();
279 // now validate the fields and return false if error
280 $valid = self
::VALID
;
281 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
282 if (!$this->_activeFields
[$i]->validate()) {
283 // no need to do any more validation
284 $erroneousField = $i;
285 $valid = self
::ERROR
;
293 * Format the field values for input to the api.
296 * (reference) associative array of name/value pairs
298 public function &getActiveFieldParams() {
300 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
301 if (isset($this->_activeFields
[$i]->_value
)
302 && !isset($params[$this->_activeFields
[$i]->_name
])
303 && !isset($this->_activeFields
[$i]->_related
)
306 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
313 * Add progress bar to the import process. Calculates time remaining, status etc.
316 * status id of the import process saved in $config->uploadDir.
317 * @param bool $startImport
318 * True when progress bar is to be initiated.
319 * @param $startTimestamp
320 * Initial timestamp when the import was started.
321 * @param $prevTimestamp
322 * Previous timestamp when this function was last called.
323 * @param $totalRowCount
324 * Total number of rows in the import file.
326 * @return NULL|$currTimestamp
328 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
329 $statusFile = CRM_Core_Config
::singleton()->uploadDir
. "status_{$statusID}.txt";
332 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
333 //do not force the browser to display the save dialog, CRM-7640
334 $contents = json_encode([0, $status]);
335 file_put_contents($statusFile, $contents);
338 $rowCount = $this->_rowCount ??
$this->_lineCount
;
339 $currTimestamp = time();
340 $time = ($currTimestamp - $prevTimestamp);
341 $recordsLeft = $totalRowCount - $rowCount;
342 if ($recordsLeft < 0) {
345 $estimatedTime = ($recordsLeft / 50) * $time;
346 $estMinutes = floor($estimatedTime / 60);
348 if ($estMinutes > 1) {
349 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
350 $estimatedTime = $estimatedTime - ($estMinutes * 60);
352 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
353 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
354 $statusMsg = ts('%1 of %2 records - %3 remaining',
355 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
357 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
358 $contents = json_encode([$processedPercent, $status]);
360 file_put_contents($statusFile, $contents);
361 return $currTimestamp;
368 public function getSelectValues(): array {
370 foreach ($this->_fields
as $name => $field) {
371 $values[$name] = $field->_title
;
379 public function getSelectTypes() {
381 foreach ($this->_fields
as $name => $field) {
382 if (isset($field->_hasLocationType
)) {
383 $values[$name] = $field->_hasLocationType
;
392 public function getHeaderPatterns() {
394 foreach ($this->_fields
as $name => $field) {
395 if (isset($field->_headerPattern
)) {
396 $values[$name] = $field->_headerPattern
;
405 public function getDataPatterns() {
407 foreach ($this->_fields
as $name => $field) {
408 $values[$name] = $field->_dataPattern
;
414 * Remove single-quote enclosures from a value array (row).
416 * @param array $values
417 * @param string $enclosure
421 public static function encloseScrub(&$values, $enclosure = "'") {
422 if (empty($values)) {
426 foreach ($values as $k => $v) {
427 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
438 public function setMaxLinesToProcess($max) {
439 $this->_maxLinesToProcess
= $max;
443 * Determines the file extension based on error code.
445 * @var int $type error code constant
448 public static function errorFileName($type) {
454 $config = CRM_Core_Config
::singleton();
455 $fileName = $config->uploadDir
. "sqlImport";
458 $fileName .= '.errors';
462 $fileName .= '.conflicts';
465 case self
::DUPLICATE
:
466 $fileName .= '.duplicates';
470 $fileName .= '.mismatch';
473 case self
::UNPARSED_ADDRESS_WARNING
:
474 $fileName .= '.unparsedAddress';
482 * Determines the file name based on error code.
484 * @var $type error code constant
487 public static function saveFileName($type) {
494 $fileName = 'Import_Errors.csv';
498 $fileName = 'Import_Conflicts.csv';
501 case self
::DUPLICATE
:
502 $fileName = 'Import_Duplicates.csv';
506 $fileName = 'Import_Mismatch.csv';
509 case self
::UNPARSED_ADDRESS_WARNING
:
510 $fileName = 'Import_Unparsed_Address.csv';
518 * Check if contact is a duplicate .
520 * @param array $formatValues
524 protected function checkContactDuplicate(&$formatValues) {
525 //retrieve contact id using contact dedupe rule
526 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->_contactType
;
527 $formatValues['version'] = 3;
528 require_once 'CRM/Utils/DeprecatedUtils.php';
529 $params = $formatValues;
530 static $cIndieFields = NULL;
531 static $defaultLocationId = NULL;
533 $contactType = $params['contact_type'];
534 if ($cIndieFields == NULL) {
535 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
536 $cIndieFields = $cTempIndieFields;
538 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
540 // set the value to default location id else set to 1
541 if (!$defaultLocationId = (int) $defaultLocation->id
) {
542 $defaultLocationId = 1;
546 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
548 $contactFormatted = [];
549 foreach ($params as $key => $field) {
550 if ($field == NULL ||
$field === '') {
553 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
554 // instead of soft credit contact.
555 if (is_array($field) && $key != "soft_credit") {
556 foreach ($field as $value) {
558 if (is_array($value)) {
559 foreach ($value as $name => $testForEmpty) {
560 if ($name !== 'phone_type' &&
561 ($testForEmpty === '' ||
$testForEmpty == NULL)
572 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
578 $value = [$key => $field];
580 // check if location related field, then we need to add primary location type
581 if (in_array($key, $locationFields)) {
582 $value['location_type_id'] = $defaultLocationId;
584 elseif (array_key_exists($key, $cIndieFields)) {
585 $value['contact_type'] = $contactType;
588 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
591 $contactFormatted['contact_type'] = $contactType;
593 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
597 * This function adds the contact variable in $values to the
598 * parameter list $params. For most cases, $values should have length 1. If
599 * the variable being added is a child of Location, a location_type_id must
600 * also be included. If it is a child of phone, a phone_type must be included.
602 * @param array $values
603 * The variable(s) to be added.
604 * @param array $params
605 * The structured parameter list.
607 * @return bool|CRM_Utils_Error
609 private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
610 // @todo - like most functions in import ... most of this is cruft....
611 // Crawl through the possible classes:
624 // Cache the various object fields
625 static $fields = NULL;
627 if ($fields == NULL) {
631 // first add core contact values since for other Civi modules they are not added
632 require_once 'CRM/Contact/BAO/Contact.php';
633 $contactFields = CRM_Contact_DAO_Contact
::fields();
634 _civicrm_api3_store_values($contactFields, $values, $params);
636 if (isset($values['contact_type'])) {
637 // we're an individual/household/org property
639 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact
::fields();
641 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
645 if (isset($values['individual_prefix'])) {
646 if (!empty($params['prefix_id'])) {
647 $prefixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'prefix_id');
648 $params['prefix'] = $prefixes[$params['prefix_id']];
651 $params['prefix'] = $values['individual_prefix'];
656 if (isset($values['individual_suffix'])) {
657 if (!empty($params['suffix_id'])) {
658 $suffixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'suffix_id');
659 $params['suffix'] = $suffixes[$params['suffix_id']];
662 $params['suffix'] = $values['individual_suffix'];
668 if (isset($values['email_greeting'])) {
669 if (!empty($params['email_greeting_id'])) {
670 $emailGreetingFilter = [
671 'contact_type' => $params['contact_type'] ??
NULL,
672 'greeting_type' => 'email_greeting',
674 $emailGreetings = CRM_Core_PseudoConstant
::greeting($emailGreetingFilter);
675 $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
678 $params['email_greeting'] = $values['email_greeting'];
684 if (isset($values['postal_greeting'])) {
685 if (!empty($params['postal_greeting_id'])) {
686 $postalGreetingFilter = [
687 'contact_type' => $params['contact_type'] ??
NULL,
688 'greeting_type' => 'postal_greeting',
690 $postalGreetings = CRM_Core_PseudoConstant
::greeting($postalGreetingFilter);
691 $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
694 $params['postal_greeting'] = $values['postal_greeting'];
699 if (isset($values['addressee'])) {
700 $params['addressee'] = $values['addressee'];
704 if (isset($values['gender'])) {
705 if (!empty($params['gender_id'])) {
706 $genders = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'gender_id');
707 $params['gender'] = $genders[$params['gender_id']];
710 $params['gender'] = $values['gender'];
715 if (!empty($values['preferred_communication_method'])) {
717 $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER
);
719 $preffComm = explode(',', $values['preferred_communication_method']);
720 foreach ($preffComm as $v) {
721 $v = strtolower(trim($v));
722 if (array_key_exists($v, $pcm)) {
727 $params['preferred_communication_method'] = $comm;
731 // format the website params.
732 if (!empty($values['url'])) {
733 static $websiteFields;
734 if (!is_array($websiteFields)) {
735 require_once 'CRM/Core/DAO/Website.php';
736 $websiteFields = CRM_Core_DAO_Website
::fields();
738 if (!array_key_exists('website', $params) ||
739 !is_array($params['website'])
741 $params['website'] = [];
744 $websiteCount = count($params['website']);
745 _civicrm_api3_store_values($websiteFields, $values,
746 $params['website'][++
$websiteCount]
752 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
753 if (!empty($values['location_type_id'])) {
754 static $fields = NULL;
755 if ($fields == NULL) {
759 foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
760 $name = strtolower($block);
761 if (!array_key_exists($name, $values)) {
765 if ($name === 'phone_ext') {
769 // block present in value array.
770 if (!array_key_exists($name, $params) ||
!is_array($params[$name])) {
774 if (!array_key_exists($block, $fields)) {
775 $className = "CRM_Core_DAO_$block";
776 $fields[$block] =& $className::fields();
779 $blockCnt = count($params[$name]);
781 // copy value to dao field name.
783 $values['name'] = $values[$name];
786 _civicrm_api3_store_values($fields[$block], $values,
787 $params[$name][++
$blockCnt]
790 if (empty($params['id']) && ($blockCnt == 1)) {
791 $params[$name][$blockCnt]['is_primary'] = TRUE;
794 // we only process single block at a time.
798 // handle address fields.
799 if (!array_key_exists('address', $params) ||
!is_array($params['address'])) {
800 $params['address'] = [];
804 foreach ($params['address'] as $cnt => $addressBlock) {
805 if (CRM_Utils_Array
::value('location_type_id', $values) ==
806 CRM_Utils_Array
::value('location_type_id', $addressBlock)
814 if (!array_key_exists('Address', $fields)) {
815 $fields['Address'] = CRM_Core_DAO_Address
::fields();
818 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
819 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
820 // the address in CRM_Core_BAO_Address::create method
821 if (!empty($values['location_type_id'])) {
822 static $customFields = [];
823 if (empty($customFields)) {
824 $customFields = CRM_Core_BAO_CustomField
::getFields('Address');
826 // make a copy of values, as we going to make changes
827 $newValues = $values;
828 foreach ($values as $key => $val) {
829 $customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key);
830 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
831 // mark an entry in fields array since we want the value of custom field to be copied
832 $fields['Address'][$key] = NULL;
834 $htmlType = $customFields[$customFieldID]['html_type'] ??
NULL;
835 if (CRM_Core_BAO_CustomField
::isSerialized($customFields[$customFieldID]) && $val) {
836 $mulValues = explode(',', $val);
837 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
838 $newValues[$key] = [];
839 foreach ($mulValues as $v1) {
840 foreach ($customOption as $v2) {
841 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
842 (strtolower($v2['value']) == strtolower(trim($v1)))
844 if ($htmlType == 'CheckBox') {
845 $newValues[$key][$v2['value']] = 1;
848 $newValues[$key][] = $v2['value'];
856 // consider new values
857 $values = $newValues;
860 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
866 'supplemental_address_1',
867 'supplemental_address_2',
868 'supplemental_address_3',
869 'StateProvince.name',
872 foreach ($addressFields as $field) {
873 if (array_key_exists($field, $values)) {
874 if (!array_key_exists('address', $params)) {
875 $params['address'] = [];
877 $params['address'][$addressCnt][$field] = $values[$field];
881 if ($addressCnt == 1) {
883 $params['address'][$addressCnt]['is_primary'] = TRUE;
888 if (isset($values['note'])) {
890 if (!isset($params['note'])) {
891 $params['note'] = [];
893 $noteBlock = count($params['note']) +
1;
895 $params['note'][$noteBlock] = [];
896 if (!isset($fields['Note'])) {
897 $fields['Note'] = CRM_Core_DAO_Note
::fields();
900 // get the current logged in civicrm user
901 $session = CRM_Core_Session
::singleton();
902 $userID = $session->get('userID');
905 $values['contact_id'] = $userID;
908 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
913 // Check for custom field values
915 if (empty($fields['custom'])) {
916 $fields['custom'] = &CRM_Core_BAO_CustomField
::getFields(CRM_Utils_Array
::value('contact_type', $values),
917 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
921 foreach ($values as $key => $value) {
922 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
923 // check if it's a valid custom field id
925 if (!array_key_exists($customFieldID, $fields['custom'])) {
926 return civicrm_api3_create_error('Invalid custom field ID');
929 $params[$key] = $value;
936 * Parse a field which could be represented by a label or name value rather than the DB value.
938 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
940 * but if not available then see if we have a label that can be converted to a name.
942 * @param string|int|null $submittedValue
943 * @param array $fieldSpec
944 * Metadata for the field
948 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
949 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
950 if (!isset($fieldSpec['bao'])) {
951 return $submittedValue;
953 /* @var \CRM_Core_DAO $bao */
954 $bao = $fieldSpec['bao'];
955 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
956 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
957 if (isset($nameOptions[$submittedValue])) {
958 return $submittedValue;
960 if (in_array($submittedValue, $nameOptions)) {
961 return array_search($submittedValue, $nameOptions, TRUE);
964 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
965 if (isset($labelOptions[$submittedValue])) {
966 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
972 * This is code extracted from 4 places where this exact snippet was being duplicated.
974 * FIXME: Extracting this was a first step, but there's also
975 * 1. Inconsistency in the way other select options are handled.
976 * Contribution adds handling for Select/Radio/Autocomplete
977 * Participant/Activity only handles Select/Radio and misses Autocomplete
978 * Membership is missing all of it
979 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
981 * @param $customFieldID
986 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
987 $mulValues = explode(',', $value);
988 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
990 foreach ($mulValues as $v1) {
991 foreach ($customOption as $customValueID => $customLabel) {
992 $customValue = $customLabel['value'];
993 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
994 (strtolower(trim($customValue)) == strtolower(trim($v1)))
996 if ($fieldType == 'CheckBox') {
997 $values[$customValue] = 1;
1000 $values[] = $customValue;
1009 * Get the ids of any contacts that match according to the rule.
1011 * @param array $formatted
1015 protected function getIdsOfMatchingContacts(array $formatted):array {
1016 // the call to the deprecated function seems to add no value other that to do an additional
1017 // check for the contact_id & type.
1018 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1019 if (!CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
1022 if (is_array($error['error_message']['params'][0])) {
1023 return $error['error_message']['params'][0];
1026 return explode(',', $error['error_message']['params'][0]);