3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
17 abstract class CRM_Import_Parser
{
21 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
26 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
31 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
34 * Codes for duplicate record handling
36 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
41 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
45 * Total number of non empty lines
48 protected $_totalCount;
51 * Running total number of valid lines
54 protected $_validCount;
57 * Running total number of invalid rows
60 protected $_invalidRowCount;
63 * Maximum number of non-empty/comment lines to process
67 protected $_maxLinesToProcess;
70 * Array of error lines, bounded by MAX_ERROR
76 * Total number of conflict lines
79 protected $_conflictCount;
82 * Array of conflict lines
85 protected $_conflicts;
88 * Total number of duplicate (from database) lines
91 protected $_duplicateCount;
94 * Array of duplicate lines
97 protected $_duplicates;
100 * Maximum number of warnings to store
103 protected $_maxWarningCount = self
::MAX_WARNINGS
;
106 * Array of warning lines, bounded by MAX_WARNING
109 protected $_warnings;
112 * Array of all the fields that could potentially be part
113 * of this import process
119 * Metadata for all available fields, keyed by unique name.
121 * This is intended to supercede $_fields which uses a special sauce format which
122 * importableFieldsMetadata uses the standard getfields type format.
126 protected $importableFieldsMetadata = [];
129 * Get metadata for all importable fields in std getfields style format.
133 public function getImportableFieldsMetadata(): array {
134 return $this->importableFieldsMetadata
;
138 * Set metadata for all importable fields in std getfields style format.
140 * @param array $importableFieldsMetadata
142 public function setImportableFieldsMetadata(array $importableFieldsMetadata): void
{
143 $this->importableFieldsMetadata
= $importableFieldsMetadata;
147 * Array of the fields that are actually part of the import process
148 * the position in the array also dictates their position in the import
152 protected $_activeFields;
155 * Cache the count of active fields
159 protected $_activeFieldCount;
162 * Cache of preview rows
169 * Filename of error data
173 protected $_errorFileName;
176 * Filename of conflict data
180 protected $_conflictFileName;
183 * Filename of duplicate data
187 protected $_duplicateFileName;
194 public $_contactType;
200 public $_contactSubType;
205 public function __construct() {
206 $this->_maxLinesToProcess
= 0;
210 * Set and validate field values.
212 * @param array $elements
214 * @param $erroneousField
219 public function setActiveFieldValues($elements, &$erroneousField = NULL) {
220 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
221 for ($i = 0; $i < $maxCount; $i++
) {
222 $this->_activeFields
[$i]->setValue($elements[$i]);
225 // reset all the values that we did not have an equivalent import element
226 for (; $i < $this->_activeFieldCount
; $i++
) {
227 $this->_activeFields
[$i]->resetValue();
230 // now validate the fields and return false if error
231 $valid = self
::VALID
;
232 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
233 if (!$this->_activeFields
[$i]->validate()) {
234 // no need to do any more validation
235 $erroneousField = $i;
236 $valid = self
::ERROR
;
244 * Format the field values for input to the api.
247 * (reference) associative array of name/value pairs
249 public function &getActiveFieldParams() {
251 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
252 if (isset($this->_activeFields
[$i]->_value
)
253 && !isset($params[$this->_activeFields
[$i]->_name
])
254 && !isset($this->_activeFields
[$i]->_related
)
257 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
264 * Add progress bar to the import process. Calculates time remaining, status etc.
267 * status id of the import process saved in $config->uploadDir.
268 * @param bool $startImport
269 * True when progress bar is to be initiated.
270 * @param $startTimestamp
271 * Initial timestamp when the import was started.
272 * @param $prevTimestamp
273 * Previous timestamp when this function was last called.
274 * @param $totalRowCount
275 * Total number of rows in the import file.
277 * @return NULL|$currTimestamp
279 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
280 $statusFile = CRM_Core_Config
::singleton()->uploadDir
. "status_{$statusID}.txt";
283 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
284 //do not force the browser to display the save dialog, CRM-7640
285 $contents = json_encode([0, $status]);
286 file_put_contents($statusFile, $contents);
289 $rowCount = $this->_rowCount ??
$this->_lineCount
;
290 $currTimestamp = time();
291 $time = ($currTimestamp - $prevTimestamp);
292 $recordsLeft = $totalRowCount - $rowCount;
293 if ($recordsLeft < 0) {
296 $estimatedTime = ($recordsLeft / 50) * $time;
297 $estMinutes = floor($estimatedTime / 60);
299 if ($estMinutes > 1) {
300 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
301 $estimatedTime = $estimatedTime - ($estMinutes * 60);
303 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
304 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
305 $statusMsg = ts('%1 of %2 records - %3 remaining',
306 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
308 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
309 $contents = json_encode([$processedPercent, $status]);
311 file_put_contents($statusFile, $contents);
312 return $currTimestamp;
319 public function getSelectValues(): array {
321 foreach ($this->_fields
as $name => $field) {
322 $values[$name] = $field->_title
;
330 public function getSelectTypes() {
332 foreach ($this->_fields
as $name => $field) {
333 if (isset($field->_hasLocationType
)) {
334 $values[$name] = $field->_hasLocationType
;
343 public function getHeaderPatterns() {
345 foreach ($this->_fields
as $name => $field) {
346 if (isset($field->_headerPattern
)) {
347 $values[$name] = $field->_headerPattern
;
356 public function getDataPatterns() {
358 foreach ($this->_fields
as $name => $field) {
359 $values[$name] = $field->_dataPattern
;
365 * Remove single-quote enclosures from a value array (row).
367 * @param array $values
368 * @param string $enclosure
372 public static function encloseScrub(&$values, $enclosure = "'") {
373 if (empty($values)) {
377 foreach ($values as $k => $v) {
378 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
389 public function setMaxLinesToProcess($max) {
390 $this->_maxLinesToProcess
= $max;
394 * Determines the file extension based on error code.
396 * @var int $type error code constant
399 public static function errorFileName($type) {
405 $config = CRM_Core_Config
::singleton();
406 $fileName = $config->uploadDir
. "sqlImport";
409 $fileName .= '.errors';
413 $fileName .= '.conflicts';
416 case self
::DUPLICATE
:
417 $fileName .= '.duplicates';
421 $fileName .= '.mismatch';
424 case self
::UNPARSED_ADDRESS_WARNING
:
425 $fileName .= '.unparsedAddress';
433 * Determines the file name based on error code.
435 * @var $type error code constant
438 public static function saveFileName($type) {
445 $fileName = 'Import_Errors.csv';
449 $fileName = 'Import_Conflicts.csv';
452 case self
::DUPLICATE
:
453 $fileName = 'Import_Duplicates.csv';
457 $fileName = 'Import_Mismatch.csv';
460 case self
::UNPARSED_ADDRESS_WARNING
:
461 $fileName = 'Import_Unparsed_Address.csv';
469 * Check if contact is a duplicate .
471 * @param array $formatValues
475 protected function checkContactDuplicate(&$formatValues) {
476 //retrieve contact id using contact dedupe rule
477 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->_contactType
;
478 $formatValues['version'] = 3;
479 require_once 'CRM/Utils/DeprecatedUtils.php';
480 $params = $formatValues;
481 static $cIndieFields = NULL;
482 static $defaultLocationId = NULL;
484 $contactType = $params['contact_type'];
485 if ($cIndieFields == NULL) {
486 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
487 $cIndieFields = $cTempIndieFields;
489 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
491 // set the value to default location id else set to 1
492 if (!$defaultLocationId = (int) $defaultLocation->id
) {
493 $defaultLocationId = 1;
497 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
499 $contactFormatted = [];
500 foreach ($params as $key => $field) {
501 if ($field == NULL ||
$field === '') {
504 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
505 // instead of soft credit contact.
506 if (is_array($field) && $key != "soft_credit") {
507 foreach ($field as $value) {
509 if (is_array($value)) {
510 foreach ($value as $name => $testForEmpty) {
511 if ($name !== 'phone_type' &&
512 ($testForEmpty === '' ||
$testForEmpty == NULL)
523 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
529 $value = [$key => $field];
531 // check if location related field, then we need to add primary location type
532 if (in_array($key, $locationFields)) {
533 $value['location_type_id'] = $defaultLocationId;
535 elseif (array_key_exists($key, $cIndieFields)) {
536 $value['contact_type'] = $contactType;
539 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
542 $contactFormatted['contact_type'] = $contactType;
544 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
548 * This function adds the contact variable in $values to the
549 * parameter list $params. For most cases, $values should have length 1. If
550 * the variable being added is a child of Location, a location_type_id must
551 * also be included. If it is a child of phone, a phone_type must be included.
553 * @param array $values
554 * The variable(s) to be added.
555 * @param array $params
556 * The structured parameter list.
558 * @return bool|CRM_Utils_Error
560 private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
561 // @todo - like most functions in import ... most of this is cruft....
562 // Crawl through the possible classes:
575 // Cache the various object fields
576 static $fields = NULL;
578 if ($fields == NULL) {
582 // first add core contact values since for other Civi modules they are not added
583 require_once 'CRM/Contact/BAO/Contact.php';
584 $contactFields = CRM_Contact_DAO_Contact
::fields();
585 _civicrm_api3_store_values($contactFields, $values, $params);
587 if (isset($values['contact_type'])) {
588 // we're an individual/household/org property
590 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact
::fields();
592 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
596 if (isset($values['individual_prefix'])) {
597 if (!empty($params['prefix_id'])) {
598 $prefixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'prefix_id');
599 $params['prefix'] = $prefixes[$params['prefix_id']];
602 $params['prefix'] = $values['individual_prefix'];
607 if (isset($values['individual_suffix'])) {
608 if (!empty($params['suffix_id'])) {
609 $suffixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'suffix_id');
610 $params['suffix'] = $suffixes[$params['suffix_id']];
613 $params['suffix'] = $values['individual_suffix'];
619 if (isset($values['email_greeting'])) {
620 if (!empty($params['email_greeting_id'])) {
621 $emailGreetingFilter = [
622 'contact_type' => $params['contact_type'] ??
NULL,
623 'greeting_type' => 'email_greeting',
625 $emailGreetings = CRM_Core_PseudoConstant
::greeting($emailGreetingFilter);
626 $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
629 $params['email_greeting'] = $values['email_greeting'];
635 if (isset($values['postal_greeting'])) {
636 if (!empty($params['postal_greeting_id'])) {
637 $postalGreetingFilter = [
638 'contact_type' => $params['contact_type'] ??
NULL,
639 'greeting_type' => 'postal_greeting',
641 $postalGreetings = CRM_Core_PseudoConstant
::greeting($postalGreetingFilter);
642 $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
645 $params['postal_greeting'] = $values['postal_greeting'];
650 if (isset($values['addressee'])) {
651 $params['addressee'] = $values['addressee'];
655 if (isset($values['gender'])) {
656 if (!empty($params['gender_id'])) {
657 $genders = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'gender_id');
658 $params['gender'] = $genders[$params['gender_id']];
661 $params['gender'] = $values['gender'];
666 if (!empty($values['preferred_communication_method'])) {
668 $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER
);
670 $preffComm = explode(',', $values['preferred_communication_method']);
671 foreach ($preffComm as $v) {
672 $v = strtolower(trim($v));
673 if (array_key_exists($v, $pcm)) {
678 $params['preferred_communication_method'] = $comm;
682 // format the website params.
683 if (!empty($values['url'])) {
684 static $websiteFields;
685 if (!is_array($websiteFields)) {
686 require_once 'CRM/Core/DAO/Website.php';
687 $websiteFields = CRM_Core_DAO_Website
::fields();
689 if (!array_key_exists('website', $params) ||
690 !is_array($params['website'])
692 $params['website'] = [];
695 $websiteCount = count($params['website']);
696 _civicrm_api3_store_values($websiteFields, $values,
697 $params['website'][++
$websiteCount]
703 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
704 if (!empty($values['location_type_id'])) {
705 static $fields = NULL;
706 if ($fields == NULL) {
710 foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
711 $name = strtolower($block);
712 if (!array_key_exists($name, $values)) {
716 if ($name === 'phone_ext') {
720 // block present in value array.
721 if (!array_key_exists($name, $params) ||
!is_array($params[$name])) {
725 if (!array_key_exists($block, $fields)) {
726 $className = "CRM_Core_DAO_$block";
727 $fields[$block] =& $className::fields();
730 $blockCnt = count($params[$name]);
732 // copy value to dao field name.
734 $values['name'] = $values[$name];
737 _civicrm_api3_store_values($fields[$block], $values,
738 $params[$name][++
$blockCnt]
741 if (empty($params['id']) && ($blockCnt == 1)) {
742 $params[$name][$blockCnt]['is_primary'] = TRUE;
745 // we only process single block at a time.
749 // handle address fields.
750 if (!array_key_exists('address', $params) ||
!is_array($params['address'])) {
751 $params['address'] = [];
755 foreach ($params['address'] as $cnt => $addressBlock) {
756 if (CRM_Utils_Array
::value('location_type_id', $values) ==
757 CRM_Utils_Array
::value('location_type_id', $addressBlock)
765 if (!array_key_exists('Address', $fields)) {
766 $fields['Address'] = CRM_Core_DAO_Address
::fields();
769 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
770 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
771 // the address in CRM_Core_BAO_Address::create method
772 if (!empty($values['location_type_id'])) {
773 static $customFields = [];
774 if (empty($customFields)) {
775 $customFields = CRM_Core_BAO_CustomField
::getFields('Address');
777 // make a copy of values, as we going to make changes
778 $newValues = $values;
779 foreach ($values as $key => $val) {
780 $customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key);
781 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
782 // mark an entry in fields array since we want the value of custom field to be copied
783 $fields['Address'][$key] = NULL;
785 $htmlType = $customFields[$customFieldID]['html_type'] ??
NULL;
786 if (CRM_Core_BAO_CustomField
::isSerialized($customFields[$customFieldID]) && $val) {
787 $mulValues = explode(',', $val);
788 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
789 $newValues[$key] = [];
790 foreach ($mulValues as $v1) {
791 foreach ($customOption as $v2) {
792 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
793 (strtolower($v2['value']) == strtolower(trim($v1)))
795 if ($htmlType == 'CheckBox') {
796 $newValues[$key][$v2['value']] = 1;
799 $newValues[$key][] = $v2['value'];
807 // consider new values
808 $values = $newValues;
811 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
817 'supplemental_address_1',
818 'supplemental_address_2',
819 'supplemental_address_3',
820 'StateProvince.name',
823 foreach ($addressFields as $field) {
824 if (array_key_exists($field, $values)) {
825 if (!array_key_exists('address', $params)) {
826 $params['address'] = [];
828 $params['address'][$addressCnt][$field] = $values[$field];
832 if ($addressCnt == 1) {
834 $params['address'][$addressCnt]['is_primary'] = TRUE;
839 if (isset($values['note'])) {
841 if (!isset($params['note'])) {
842 $params['note'] = [];
844 $noteBlock = count($params['note']) +
1;
846 $params['note'][$noteBlock] = [];
847 if (!isset($fields['Note'])) {
848 $fields['Note'] = CRM_Core_DAO_Note
::fields();
851 // get the current logged in civicrm user
852 $session = CRM_Core_Session
::singleton();
853 $userID = $session->get('userID');
856 $values['contact_id'] = $userID;
859 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
864 // Check for custom field values
866 if (empty($fields['custom'])) {
867 $fields['custom'] = &CRM_Core_BAO_CustomField
::getFields(CRM_Utils_Array
::value('contact_type', $values),
868 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
872 foreach ($values as $key => $value) {
873 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
874 // check if it's a valid custom field id
876 if (!array_key_exists($customFieldID, $fields['custom'])) {
877 return civicrm_api3_create_error('Invalid custom field ID');
880 $params[$key] = $value;
887 * Parse a field which could be represented by a label or name value rather than the DB value.
889 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
891 * but if not available then see if we have a label that can be converted to a name.
893 * @param string|int|null $submittedValue
894 * @param array $fieldSpec
895 * Metadata for the field
899 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
900 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
901 if (!isset($fieldSpec['bao'])) {
902 return $submittedValue;
904 /* @var \CRM_Core_DAO $bao */
905 $bao = $fieldSpec['bao'];
906 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
907 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
908 if (isset($nameOptions[$submittedValue])) {
909 return $submittedValue;
911 if (in_array($submittedValue, $nameOptions)) {
912 return array_search($submittedValue, $nameOptions, TRUE);
915 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
916 if (isset($labelOptions[$submittedValue])) {
917 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
923 * This is code extracted from 4 places where this exact snippet was being duplicated.
925 * FIXME: Extracting this was a first step, but there's also
926 * 1. Inconsistency in the way other select options are handled.
927 * Contribution adds handling for Select/Radio/Autocomplete
928 * Participant/Activity only handles Select/Radio and misses Autocomplete
929 * Membership is missing all of it
930 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
932 * @param $customFieldID
937 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
938 $mulValues = explode(',', $value);
939 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
941 foreach ($mulValues as $v1) {
942 foreach ($customOption as $customValueID => $customLabel) {
943 $customValue = $customLabel['value'];
944 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
945 (strtolower(trim($customValue)) == strtolower(trim($v1)))
947 $values[] = $customValue;
955 * Get the ids of any contacts that match according to the rule.
957 * @param array $formatted
961 protected function getIdsOfMatchingContacts(array $formatted):array {
962 // the call to the deprecated function seems to add no value other that to do an additional
963 // check for the contact_id & type.
964 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
965 if (!CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
968 if (is_array($error['error_message']['params'][0])) {
969 return $error['error_message']['params'][0];
972 return explode(',', $error['error_message']['params'][0]);