3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
12 use Civi\Api4\UserJob
;
17 * @copyright CiviCRM LLC https://civicrm.org/licensing
19 abstract class CRM_Import_Parser
{
23 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
28 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
33 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
36 * Codes for duplicate record handling
38 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
43 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
49 * This is the primary key of the civicrm_user_job table which is used to
59 public function getUserJobID(): ?
int {
60 return $this->userJobID
;
66 * @param int $userJobID
68 public function setUserJobID(int $userJobID): void
{
69 $this->userJobID
= $userJobID;
75 * API call to retrieve the userJob row.
79 * @throws \API_Exception
81 protected function getUserJob(): array {
83 ->addWhere('id', '=', $this->getUserJobID())
89 * Get the submitted value, as stored on the user job.
91 * @param string $fieldName
95 * @throws \API_Exception
97 protected function getSubmittedValue(string $fieldName) {
98 return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
102 * Get configured contact type.
104 * @throws \API_Exception
106 protected function getContactType() {
107 if (!$this->_contactType
) {
108 $contactTypeMapping = [
109 CRM_Import_Parser
::CONTACT_INDIVIDUAL
=> 'Individual',
110 CRM_Import_Parser
::CONTACT_HOUSEHOLD
=> 'Household',
111 CRM_Import_Parser
::CONTACT_ORGANIZATION
=> 'Organization',
113 $this->_contactType
= $contactTypeMapping[$this->getSubmittedValue('contactType')];
115 return $this->_contactType
;
119 * Get configured contact type.
121 * @return string|null
123 * @throws \API_Exception
125 public function getContactSubType() {
126 if (!$this->_contactSubType
) {
127 $this->_contactSubType
= $this->getSubmittedValue('contactSubType');
129 return $this->_contactSubType
;
133 * Total number of non empty lines
136 protected $_totalCount;
139 * Running total number of valid lines
142 protected $_validCount;
145 * Running total number of invalid rows
148 protected $_invalidRowCount;
151 * Maximum number of non-empty/comment lines to process
155 protected $_maxLinesToProcess;
158 * Array of error lines, bounded by MAX_ERROR
164 * Total number of conflict lines
167 protected $_conflictCount;
170 * Array of conflict lines
173 protected $_conflicts;
176 * Total number of duplicate (from database) lines
179 protected $_duplicateCount;
182 * Array of duplicate lines
185 protected $_duplicates;
188 * Maximum number of warnings to store
191 protected $_maxWarningCount = self
::MAX_WARNINGS
;
194 * Array of warning lines, bounded by MAX_WARNING
197 protected $_warnings;
200 * Array of all the fields that could potentially be part
201 * of this import process
207 * Metadata for all available fields, keyed by unique name.
209 * This is intended to supercede $_fields which uses a special sauce format which
210 * importableFieldsMetadata uses the standard getfields type format.
214 protected $importableFieldsMetadata = [];
217 * Get metadata for all importable fields in std getfields style format.
221 public function getImportableFieldsMetadata(): array {
222 return $this->importableFieldsMetadata
;
226 * Set metadata for all importable fields in std getfields style format.
228 * @param array $importableFieldsMetadata
230 public function setImportableFieldsMetadata(array $importableFieldsMetadata): void
{
231 $this->importableFieldsMetadata
= $importableFieldsMetadata;
235 * Array of the fields that are actually part of the import process
236 * the position in the array also dictates their position in the import
240 protected $_activeFields;
243 * Cache the count of active fields
247 protected $_activeFieldCount;
250 * Cache of preview rows
257 * Filename of error data
261 protected $_errorFileName;
264 * Filename of conflict data
268 protected $_conflictFileName;
271 * Filename of duplicate data
275 protected $_duplicateFileName;
282 public $_contactType;
285 * @param string $contactType
287 * @return CRM_Import_Parser
289 public function setContactType(string $contactType): CRM_Import_Parser
{
290 $this->_contactType
= $contactType;
299 public $_contactSubType;
302 * @param int|null $contactSubType
306 public function setContactSubType(?
int $contactSubType): self
{
307 $this->_contactSubType
= $contactSubType;
314 public function __construct() {
315 $this->_maxLinesToProcess
= 0;
319 * Set and validate field values.
321 * @param array $elements
324 public function setActiveFieldValues($elements): void
{
325 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
326 for ($i = 0; $i < $maxCount; $i++
) {
327 $this->_activeFields
[$i]->setValue($elements[$i]);
330 // reset all the values that we did not have an equivalent import element
331 for (; $i < $this->_activeFieldCount
; $i++
) {
332 $this->_activeFields
[$i]->resetValue();
337 * Format the field values for input to the api.
340 * (reference) associative array of name/value pairs
342 public function &getActiveFieldParams() {
344 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
345 if (isset($this->_activeFields
[$i]->_value
)
346 && !isset($params[$this->_activeFields
[$i]->_name
])
347 && !isset($this->_activeFields
[$i]->_related
)
350 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
357 * Add progress bar to the import process. Calculates time remaining, status etc.
360 * status id of the import process saved in $config->uploadDir.
361 * @param bool $startImport
362 * True when progress bar is to be initiated.
363 * @param $startTimestamp
364 * Initial timestamp when the import was started.
365 * @param $prevTimestamp
366 * Previous timestamp when this function was last called.
367 * @param $totalRowCount
368 * Total number of rows in the import file.
370 * @return NULL|$currTimestamp
372 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
373 $statusFile = CRM_Core_Config
::singleton()->uploadDir
. "status_{$statusID}.txt";
376 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
377 //do not force the browser to display the save dialog, CRM-7640
378 $contents = json_encode([0, $status]);
379 file_put_contents($statusFile, $contents);
382 $rowCount = $this->_rowCount ??
$this->_lineCount
;
383 $currTimestamp = time();
384 $time = ($currTimestamp - $prevTimestamp);
385 $recordsLeft = $totalRowCount - $rowCount;
386 if ($recordsLeft < 0) {
389 $estimatedTime = ($recordsLeft / 50) * $time;
390 $estMinutes = floor($estimatedTime / 60);
392 if ($estMinutes > 1) {
393 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
394 $estimatedTime = $estimatedTime - ($estMinutes * 60);
396 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
397 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
398 $statusMsg = ts('%1 of %2 records - %3 remaining',
399 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
401 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
402 $contents = json_encode([$processedPercent, $status]);
404 file_put_contents($statusFile, $contents);
405 return $currTimestamp;
412 public function getSelectValues(): array {
414 foreach ($this->_fields
as $name => $field) {
415 $values[$name] = $field->_title
;
423 public function getSelectTypes() {
425 foreach ($this->_fields
as $name => $field) {
426 if (isset($field->_hasLocationType
)) {
427 $values[$name] = $field->_hasLocationType
;
436 public function getHeaderPatterns() {
438 foreach ($this->_fields
as $name => $field) {
439 if (isset($field->_headerPattern
)) {
440 $values[$name] = $field->_headerPattern
;
449 public function getDataPatterns() {
451 foreach ($this->_fields
as $name => $field) {
452 $values[$name] = $field->_dataPattern
;
458 * Remove single-quote enclosures from a value array (row).
460 * @param array $values
461 * @param string $enclosure
465 public static function encloseScrub(&$values, $enclosure = "'") {
466 if (empty($values)) {
470 foreach ($values as $k => $v) {
471 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
482 public function setMaxLinesToProcess($max) {
483 $this->_maxLinesToProcess
= $max;
487 * Determines the file extension based on error code.
489 * @var int $type error code constant
492 public static function errorFileName($type) {
498 $config = CRM_Core_Config
::singleton();
499 $fileName = $config->uploadDir
. "sqlImport";
502 $fileName .= '.errors';
506 $fileName .= '.conflicts';
509 case self
::DUPLICATE
:
510 $fileName .= '.duplicates';
514 $fileName .= '.mismatch';
517 case self
::UNPARSED_ADDRESS_WARNING
:
518 $fileName .= '.unparsedAddress';
526 * Determines the file name based on error code.
528 * @var $type error code constant
531 public static function saveFileName($type) {
538 $fileName = 'Import_Errors.csv';
542 $fileName = 'Import_Conflicts.csv';
545 case self
::DUPLICATE
:
546 $fileName = 'Import_Duplicates.csv';
550 $fileName = 'Import_Mismatch.csv';
553 case self
::UNPARSED_ADDRESS_WARNING
:
554 $fileName = 'Import_Unparsed_Address.csv';
562 * Check if contact is a duplicate .
564 * @param array $formatValues
568 protected function checkContactDuplicate(&$formatValues) {
569 //retrieve contact id using contact dedupe rule
570 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->_contactType
;
571 $formatValues['version'] = 3;
572 require_once 'CRM/Utils/DeprecatedUtils.php';
573 $params = $formatValues;
574 static $cIndieFields = NULL;
575 static $defaultLocationId = NULL;
577 $contactType = $params['contact_type'];
578 if ($cIndieFields == NULL) {
579 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
580 $cIndieFields = $cTempIndieFields;
582 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
584 // set the value to default location id else set to 1
585 if (!$defaultLocationId = (int) $defaultLocation->id
) {
586 $defaultLocationId = 1;
590 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
592 $contactFormatted = [];
593 foreach ($params as $key => $field) {
594 if ($field == NULL ||
$field === '') {
597 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
598 // instead of soft credit contact.
599 if (is_array($field) && $key != "soft_credit") {
600 foreach ($field as $value) {
602 if (is_array($value)) {
603 foreach ($value as $name => $testForEmpty) {
604 if ($name !== 'phone_type' &&
605 ($testForEmpty === '' ||
$testForEmpty == NULL)
616 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
622 $value = [$key => $field];
624 // check if location related field, then we need to add primary location type
625 if (in_array($key, $locationFields)) {
626 $value['location_type_id'] = $defaultLocationId;
628 elseif (array_key_exists($key, $cIndieFields)) {
629 $value['contact_type'] = $contactType;
632 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
635 $contactFormatted['contact_type'] = $contactType;
637 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
641 * This function adds the contact variable in $values to the
642 * parameter list $params. For most cases, $values should have length 1. If
643 * the variable being added is a child of Location, a location_type_id must
644 * also be included. If it is a child of phone, a phone_type must be included.
646 * @param array $values
647 * The variable(s) to be added.
648 * @param array $params
649 * The structured parameter list.
651 * @return bool|CRM_Utils_Error
653 private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
654 // @todo - like most functions in import ... most of this is cruft....
655 // Crawl through the possible classes:
668 // Cache the various object fields
669 static $fields = NULL;
671 if ($fields == NULL) {
675 // first add core contact values since for other Civi modules they are not added
676 require_once 'CRM/Contact/BAO/Contact.php';
677 $contactFields = CRM_Contact_DAO_Contact
::fields();
678 _civicrm_api3_store_values($contactFields, $values, $params);
680 if (isset($values['contact_type'])) {
681 // we're an individual/household/org property
683 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact
::fields();
685 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
689 if (isset($values['individual_prefix'])) {
690 if (!empty($params['prefix_id'])) {
691 $prefixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'prefix_id');
692 $params['prefix'] = $prefixes[$params['prefix_id']];
695 $params['prefix'] = $values['individual_prefix'];
700 if (isset($values['individual_suffix'])) {
701 if (!empty($params['suffix_id'])) {
702 $suffixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'suffix_id');
703 $params['suffix'] = $suffixes[$params['suffix_id']];
706 $params['suffix'] = $values['individual_suffix'];
712 if (isset($values['email_greeting'])) {
713 if (!empty($params['email_greeting_id'])) {
714 $emailGreetingFilter = [
715 'contact_type' => $params['contact_type'] ??
NULL,
716 'greeting_type' => 'email_greeting',
718 $emailGreetings = CRM_Core_PseudoConstant
::greeting($emailGreetingFilter);
719 $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
722 $params['email_greeting'] = $values['email_greeting'];
728 if (isset($values['postal_greeting'])) {
729 if (!empty($params['postal_greeting_id'])) {
730 $postalGreetingFilter = [
731 'contact_type' => $params['contact_type'] ??
NULL,
732 'greeting_type' => 'postal_greeting',
734 $postalGreetings = CRM_Core_PseudoConstant
::greeting($postalGreetingFilter);
735 $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
738 $params['postal_greeting'] = $values['postal_greeting'];
743 if (isset($values['addressee'])) {
744 $params['addressee'] = $values['addressee'];
748 if (isset($values['gender'])) {
749 if (!empty($params['gender_id'])) {
750 $genders = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'gender_id');
751 $params['gender'] = $genders[$params['gender_id']];
754 $params['gender'] = $values['gender'];
759 if (!empty($values['preferred_communication_method'])) {
761 $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER
);
763 $preffComm = explode(',', $values['preferred_communication_method']);
764 foreach ($preffComm as $v) {
765 $v = strtolower(trim($v));
766 if (array_key_exists($v, $pcm)) {
771 $params['preferred_communication_method'] = $comm;
775 // format the website params.
776 if (!empty($values['url'])) {
777 static $websiteFields;
778 if (!is_array($websiteFields)) {
779 require_once 'CRM/Core/DAO/Website.php';
780 $websiteFields = CRM_Core_DAO_Website
::fields();
782 if (!array_key_exists('website', $params) ||
783 !is_array($params['website'])
785 $params['website'] = [];
788 $websiteCount = count($params['website']);
789 _civicrm_api3_store_values($websiteFields, $values,
790 $params['website'][++
$websiteCount]
796 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
797 if (!empty($values['location_type_id'])) {
798 static $fields = NULL;
799 if ($fields == NULL) {
803 foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
804 $name = strtolower($block);
805 if (!array_key_exists($name, $values)) {
809 if ($name === 'phone_ext') {
813 // block present in value array.
814 if (!array_key_exists($name, $params) ||
!is_array($params[$name])) {
818 if (!array_key_exists($block, $fields)) {
819 $className = "CRM_Core_DAO_$block";
820 $fields[$block] =& $className::fields();
823 $blockCnt = count($params[$name]);
825 // copy value to dao field name.
827 $values['name'] = $values[$name];
830 _civicrm_api3_store_values($fields[$block], $values,
831 $params[$name][++
$blockCnt]
834 if (empty($params['id']) && ($blockCnt == 1)) {
835 $params[$name][$blockCnt]['is_primary'] = TRUE;
838 // we only process single block at a time.
842 // handle address fields.
843 if (!array_key_exists('address', $params) ||
!is_array($params['address'])) {
844 $params['address'] = [];
848 foreach ($params['address'] as $cnt => $addressBlock) {
849 if (CRM_Utils_Array
::value('location_type_id', $values) ==
850 CRM_Utils_Array
::value('location_type_id', $addressBlock)
858 if (!array_key_exists('Address', $fields)) {
859 $fields['Address'] = CRM_Core_DAO_Address
::fields();
862 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
863 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
864 // the address in CRM_Core_BAO_Address::create method
865 if (!empty($values['location_type_id'])) {
866 static $customFields = [];
867 if (empty($customFields)) {
868 $customFields = CRM_Core_BAO_CustomField
::getFields('Address');
870 // make a copy of values, as we going to make changes
871 $newValues = $values;
872 foreach ($values as $key => $val) {
873 $customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key);
874 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
875 // mark an entry in fields array since we want the value of custom field to be copied
876 $fields['Address'][$key] = NULL;
878 $htmlType = $customFields[$customFieldID]['html_type'] ??
NULL;
879 if (CRM_Core_BAO_CustomField
::isSerialized($customFields[$customFieldID]) && $val) {
880 $mulValues = explode(',', $val);
881 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
882 $newValues[$key] = [];
883 foreach ($mulValues as $v1) {
884 foreach ($customOption as $v2) {
885 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
886 (strtolower($v2['value']) == strtolower(trim($v1)))
888 if ($htmlType == 'CheckBox') {
889 $newValues[$key][$v2['value']] = 1;
892 $newValues[$key][] = $v2['value'];
900 // consider new values
901 $values = $newValues;
904 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
910 'supplemental_address_1',
911 'supplemental_address_2',
912 'supplemental_address_3',
913 'StateProvince.name',
916 foreach ($addressFields as $field) {
917 if (array_key_exists($field, $values)) {
918 if (!array_key_exists('address', $params)) {
919 $params['address'] = [];
921 $params['address'][$addressCnt][$field] = $values[$field];
925 if ($addressCnt == 1) {
927 $params['address'][$addressCnt]['is_primary'] = TRUE;
932 if (isset($values['note'])) {
934 if (!isset($params['note'])) {
935 $params['note'] = [];
937 $noteBlock = count($params['note']) +
1;
939 $params['note'][$noteBlock] = [];
940 if (!isset($fields['Note'])) {
941 $fields['Note'] = CRM_Core_DAO_Note
::fields();
944 // get the current logged in civicrm user
945 $session = CRM_Core_Session
::singleton();
946 $userID = $session->get('userID');
949 $values['contact_id'] = $userID;
952 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
957 // Check for custom field values
959 if (empty($fields['custom'])) {
960 $fields['custom'] = &CRM_Core_BAO_CustomField
::getFields(CRM_Utils_Array
::value('contact_type', $values),
961 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
965 foreach ($values as $key => $value) {
966 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
967 // check if it's a valid custom field id
969 if (!array_key_exists($customFieldID, $fields['custom'])) {
970 return civicrm_api3_create_error('Invalid custom field ID');
973 $params[$key] = $value;
980 * Parse a field which could be represented by a label or name value rather than the DB value.
982 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
984 * but if not available then see if we have a label that can be converted to a name.
986 * @param string|int|null $submittedValue
987 * @param array $fieldSpec
988 * Metadata for the field
992 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
993 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
994 if (!isset($fieldSpec['bao'])) {
995 return $submittedValue;
997 /* @var \CRM_Core_DAO $bao */
998 $bao = $fieldSpec['bao'];
999 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1000 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1001 if (isset($nameOptions[$submittedValue])) {
1002 return $submittedValue;
1004 if (in_array($submittedValue, $nameOptions)) {
1005 return array_search($submittedValue, $nameOptions, TRUE);
1008 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1009 if (isset($labelOptions[$submittedValue])) {
1010 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1016 * This is code extracted from 4 places where this exact snippet was being duplicated.
1018 * FIXME: Extracting this was a first step, but there's also
1019 * 1. Inconsistency in the way other select options are handled.
1020 * Contribution adds handling for Select/Radio/Autocomplete
1021 * Participant/Activity only handles Select/Radio and misses Autocomplete
1022 * Membership is missing all of it
1023 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1025 * @param $customFieldID
1030 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1031 $mulValues = explode(',', $value);
1032 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1034 foreach ($mulValues as $v1) {
1035 foreach ($customOption as $customValueID => $customLabel) {
1036 $customValue = $customLabel['value'];
1037 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1038 (strtolower(trim($customValue)) == strtolower(trim($v1)))
1040 $values[] = $customValue;
1048 * Get the ids of any contacts that match according to the rule.
1050 * @param array $formatted
1054 protected function getIdsOfMatchingContacts(array $formatted):array {
1055 // the call to the deprecated function seems to add no value other that to do an additional
1056 // check for the contact_id & type.
1057 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($formatted);
1058 if (!CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
1061 if (is_array($error['error_message']['params'][0])) {
1062 return $error['error_message']['params'][0];
1065 return explode(',', $error['error_message']['params'][0]);