3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
12 use Civi\Api4\Campaign
;
13 use Civi\Api4\CustomField
;
15 use Civi\Api4\UserJob
;
20 * @copyright CiviCRM LLC https://civicrm.org/licensing
22 abstract class CRM_Import_Parser
{
26 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
31 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
36 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
39 * Codes for duplicate record handling
41 const DUPLICATE_SKIP
= 1, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
46 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
51 * This is the primary key of the civicrm_user_job table which is used to
59 * The user job in use.
66 * Potentially ambiguous options.
68 * For example 'UT' is a state in more than one country.
72 protected $ambiguousOptions = [];
75 * States to country mapping.
79 protected $statesByCountry = [];
84 public function getUserJobID(): ?
int {
85 return $this->userJobID
;
89 * Ids of contacts created this iteration.
93 protected $createdContacts = [];
98 * @param int $userJobID
102 public function setUserJobID(int $userJobID): self
{
103 $this->userJobID
= $userJobID;
108 * Countries that the site is restricted to
112 private $availableCountries;
118 public function getTrackingFields(): array {
125 * API call to retrieve the userJob row.
129 * @throws \API_Exception
131 protected function getUserJob(): array {
132 if (empty($this->userJob
)) {
133 $this->userJob
= UserJob
::get()
134 ->addWhere('id', '=', $this->getUserJobID())
138 return $this->userJob
;
142 * Get the relevant datasource object.
144 * @return \CRM_Import_DataSource|null
146 * @throws \API_Exception
148 protected function getDataSourceObject(): ?CRM_Import_DataSource
{
149 $className = $this->getSubmittedValue('dataSource');
151 /* @var CRM_Import_DataSource $dataSource */
152 return new $className($this->getUserJobID());
158 * Get the submitted value, as stored on the user job.
160 * @param string $fieldName
164 * @noinspection PhpDocMissingThrowsInspection
165 * @noinspection PhpUnhandledExceptionInspection
167 protected function getSubmittedValue(string $fieldName) {
168 return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
172 * Has the import completed.
176 * @throws \API_Exception
177 * @throws \CRM_Core_Exception
179 public function isComplete() :bool {
180 return $this->getDataSourceObject()->isCompleted();
184 * Get configured contact type.
188 protected function getContactType(): string {
189 if (!$this->_contactType
) {
190 $contactTypeMapping = [
191 CRM_Import_Parser
::CONTACT_INDIVIDUAL
=> 'Individual',
192 CRM_Import_Parser
::CONTACT_HOUSEHOLD
=> 'Household',
193 CRM_Import_Parser
::CONTACT_ORGANIZATION
=> 'Organization',
195 $this->_contactType
= $contactTypeMapping[$this->getSubmittedValue('contactType')];
197 return $this->_contactType
;
201 * Get configured contact type.
203 * @return string|null
205 public function getContactSubType(): ?
string {
206 if (!$this->_contactSubType
) {
207 $this->_contactSubType
= $this->getSubmittedValue('contactSubType');
209 return $this->_contactSubType
;
213 * Total number of non empty lines
216 protected $_totalCount;
219 * Running total number of valid lines
222 protected $_validCount;
225 * Running total number of invalid rows
228 protected $_invalidRowCount;
231 * Maximum number of non-empty/comment lines to process
235 protected $_maxLinesToProcess;
238 * Array of error lines, bounded by MAX_ERROR
244 * Total number of duplicate (from database) lines
247 protected $_duplicateCount;
250 * Array of duplicate lines
253 protected $_duplicates;
256 * Maximum number of warnings to store
259 protected $_maxWarningCount = self
::MAX_WARNINGS
;
262 * Array of warning lines, bounded by MAX_WARNING
265 protected $_warnings;
268 * Array of all the fields that could potentially be part
269 * of this import process
275 * Metadata for all available fields, keyed by unique name.
277 * This is intended to supercede $_fields which uses a special sauce format which
278 * importableFieldsMetadata uses the standard getfields type format.
282 protected $importableFieldsMetadata = [];
285 * Get metadata for all importable fields in std getfields style format.
289 public function getImportableFieldsMetadata(): array {
290 return $this->importableFieldsMetadata
;
294 * Set metadata for all importable fields in std getfields style format.
296 * @param array $importableFieldsMetadata
298 public function setImportableFieldsMetadata(array $importableFieldsMetadata): void
{
299 $this->importableFieldsMetadata
= $importableFieldsMetadata;
303 * Gets the fields available for importing in a key-name, title format.
306 * eg. ['first_name' => 'First Name'.....]
308 * @throws \API_Exception
310 * @todo - we are constructing the metadata before we
311 * have set the contact type so we re-do it here.
313 * Once we have cleaned up the way the mapper is handled
314 * we can ditch all the existing _construct parameters in favour
315 * of just the userJobID - there are current open PRs towards this end.
317 public function getAvailableFields(): array {
318 $this->setFieldMetadata();
320 foreach ($this->getImportableFieldsMetadata() as $name => $field) {
321 if ($name === 'id' && $this->isSkipDuplicates()) {
322 // Duplicates are being skipped so id matching is not availble.
325 $return[$name] = $field['html']['label'] ??
$field['title'];
331 * Did the user specify duplicates should be skipped and not imported.
335 * @throws \API_Exception
337 protected function isSkipDuplicates(): bool {
338 return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser
::DUPLICATE_SKIP
;
342 * Is this a case where the user has opted to update existing contacts.
346 protected function isUpdateExisting(): bool {
347 return in_array((int) $this->getSubmittedValue('onDuplicate'), [
348 CRM_Import_Parser
::DUPLICATE_UPDATE
,
349 CRM_Import_Parser
::DUPLICATE_FILL
,
354 * Did the user specify duplicates checking should be skipped, resulting in possible duplicate contacts.
356 * Note we still need to check for external_identifier as it will hard-fail
361 protected function isIgnoreDuplicates(): bool {
362 return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser
::DUPLICATE_NOCHECK
;
366 * Did the user specify duplicates should be filled with missing data.
370 protected function isFillDuplicates(): bool {
371 return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser
::DUPLICATE_FILL
;
375 * Array of the fields that are actually part of the import process
376 * the position in the array also dictates their position in the import
380 protected $_activeFields = [];
383 * Cache the count of active fields
387 protected $_activeFieldCount;
390 * Cache of preview rows
397 * Filename of error data
401 protected $_errorFileName;
404 * Filename of duplicate data
408 protected $_duplicateFileName;
415 public $_contactType;
418 * @param string $contactType
420 * @return CRM_Import_Parser
422 public function setContactType(string $contactType): CRM_Import_Parser
{
423 $this->_contactType
= $contactType;
432 public $_contactSubType;
435 * @param int|null $contactSubType
439 public function setContactSubType(?
int $contactSubType): self
{
440 $this->_contactSubType
= $contactSubType;
447 public function __construct() {
448 $this->_maxLinesToProcess
= 0;
452 * Set and validate field values.
454 * @param array $elements
457 public function setActiveFieldValues($elements): void
{
458 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
459 for ($i = 0; $i < $maxCount; $i++
) {
460 $this->_activeFields
[$i]->setValue($elements[$i]);
463 // reset all the values that we did not have an equivalent import element
464 for (; $i < $this->_activeFieldCount
; $i++
) {
465 $this->_activeFields
[$i]->resetValue();
470 * Format the field values for input to the api.
473 * (reference) associative array of name/value pairs
475 public function &getActiveFieldParams() {
477 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
478 if (isset($this->_activeFields
[$i]->_value
)
479 && !isset($params[$this->_activeFields
[$i]->_name
])
480 && !isset($this->_activeFields
[$i]->_related
)
483 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
490 * Add progress bar to the import process. Calculates time remaining, status etc.
493 * status id of the import process saved in $config->uploadDir.
494 * @param bool $startImport
495 * True when progress bar is to be initiated.
496 * @param $startTimestamp
497 * Initial timestamp when the import was started.
498 * @param $prevTimestamp
499 * Previous timestamp when this function was last called.
500 * @param $totalRowCount
501 * Total number of rows in the import file.
503 * @return NULL|$currTimestamp
505 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
506 $statusFile = CRM_Core_Config
::singleton()->uploadDir
. "status_{$statusID}.txt";
509 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
510 //do not force the browser to display the save dialog, CRM-7640
511 $contents = json_encode([0, $status]);
512 file_put_contents($statusFile, $contents);
515 $rowCount = $this->_rowCount ??
$this->_lineCount
;
516 $currTimestamp = time();
517 $time = ($currTimestamp - $prevTimestamp);
518 $recordsLeft = $totalRowCount - $rowCount;
519 if ($recordsLeft < 0) {
522 $estimatedTime = ($recordsLeft / 50) * $time;
523 $estMinutes = floor($estimatedTime / 60);
525 if ($estMinutes > 1) {
526 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
527 $estimatedTime = $estimatedTime - ($estMinutes * 60);
529 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
530 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
531 $statusMsg = ts('%1 of %2 records - %3 remaining',
532 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
534 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
535 $contents = json_encode([$processedPercent, $status]);
537 file_put_contents($statusFile, $contents);
538 return $currTimestamp;
545 public function getSelectValues(): array {
547 foreach ($this->_fields
as $name => $field) {
548 $values[$name] = $field->_title
;
556 public function getSelectTypes() {
558 // This is only called from the MapField form in isolation now,
559 // so we need to set the metadata.
561 foreach ($this->_fields
as $name => $field) {
562 if (isset($field->_hasLocationType
)) {
563 $values[$name] = $field->_hasLocationType
;
572 public function getHeaderPatterns(): array {
574 foreach ($this->importableFieldsMetadata
as $name => $field) {
575 if (isset($field['headerPattern'])) {
576 $values[$name] = $field['headerPattern'] ?
: '//';
585 public function getDataPatterns():array {
587 foreach ($this->_fields
as $name => $field) {
588 $values[$name] = $field->_dataPattern
;
594 * Remove single-quote enclosures from a value array (row).
596 * @param array $values
597 * @param string $enclosure
601 public static function encloseScrub(&$values, $enclosure = "'") {
602 if (empty($values)) {
606 foreach ($values as $k => $v) {
607 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
618 public function setMaxLinesToProcess($max) {
619 $this->_maxLinesToProcess
= $max;
623 * Validate that we have the required fields to create the contact or find it to update.
625 * Note that the users duplicate selection affects this as follows
626 * - if they did not select an update variant then the id field is not
627 * permitted in the mapping - so we can assume the presence of id means
629 * - the external_identifier field is valid in place of the other fields
630 * when they have chosen update or fill - in this case we are only looking
631 * to update an existing contact.
633 * @param string $contactType
634 * @param array $params
635 * @param bool $isPermitExistingMatchFields
636 * True if the it is enough to have fields which will enable us to find
637 * an existing contact (eg. external_identifier).
638 * @param string $prefixString
639 * String to include in the exception (e.g '(Child of)' if we are validating
643 * @throws \CRM_Core_Exception
645 protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void
{
646 if (!empty($params['id'])) {
651 'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
652 'email' => ts('Email Address'),
654 'Organization' => ['organization_name' => ts('Organization Name')],
655 'Household' => ['household_name' => ts('Household Name')],
657 if ($isPermitExistingMatchFields) {
658 $requiredFields['external_identifier'] = ts('External Identifier');
659 // Historically just an email has been accepted as it is 'usually good enough'
660 // for a dedupe rule look up - but really this is a stand in for
661 // whatever is needed to find an existing matching contact using the
662 // specified dedupe rule (or the default Unsupervised if not specified).
663 $requiredFields['email'] = ts('Email Address');
665 $this->validateRequiredFields($requiredFields, $params, $prefixString);
668 protected function doPostImportActions() {
669 $userJob = $this->getUserJob();
670 $summaryInfo = $userJob['metadata']['summary_info'] ??
[];
671 $actions = $userJob['metadata']['post_actions'] ??
[];
672 if (!empty($actions['group'])) {
673 $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts
, $actions['group']);
674 foreach ($actions['group'] as $groupID) {
675 $summaryInfo['groups'][$groupID]['added'] +
= $groupAdditions[$groupID]['added'];
676 $summaryInfo['groups'][$groupID]['notAdded'] +
= $groupAdditions[$groupID]['notAdded'];
679 if (!empty($actions['tag'])) {
680 $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts
, $actions['tag']);
681 foreach ($actions['tag'] as $tagID) {
682 $summaryInfo['tags'][$tagID]['added'] +
= $tagAdditions[$tagID]['added'];
683 $summaryInfo['tags'][$tagID]['notAdded'] +
= $tagAdditions[$tagID]['notAdded'];
687 $this->userJob
['metadata']['summary_info'] = $summaryInfo;
688 UserJob
::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob
['metadata']])->execute();
691 public function queue() {
692 $dataSource = $this->getDataSourceObject();
693 $totalRowCount = $totalRows = $dataSource->getRowCount(['new']);
694 $queue = Civi
::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']);
697 while ($totalRows > 0) {
698 if ($totalRows < $batchSize) {
699 $batchSize = $totalRows;
701 $task = new CRM_Queue_Task(
702 [get_class($this), 'runImport'],
703 ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize],
704 ts('Processed %1 rows out of %2', [1 => $offset +
$batchSize, 2 => $totalRowCount])
706 $queue->createItem($task);
707 $totalRows -= $batchSize;
708 $offset +
= $batchSize;
714 * Add imported contacts to groups.
716 * @param array $contactIDs
717 * @param array $groups
721 private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array {
722 $groupAdditions = [];
723 foreach ($groups as $groupID) {
724 // @todo - this function has been in use historically but it does not seem
725 // to add much efficiency of get + create api calls
726 // and it doesn't give enough control over cache flushing for smaller batches.
727 // Note that the import updates a lot of enities & checking & updating the group
728 // shouldn't add much performance wise. However, cache flushing will
729 $addCount = CRM_Contact_BAO_GroupContact
::addContactsToGroup($contactIDs, $groupID);
730 $groupAdditions[$groupID] = [
731 'added' => (int) $addCount[1],
732 'notAdded' => (int) $addCount[2],
735 return $groupAdditions;
739 * Tag imported contacts.
741 * @param array $contactIDs
746 private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) {
748 foreach ($tags as $tagID) {
749 // @todo - this function has been in use historically but it does not seem
750 // to add much efficiency of get + create api calls
751 // and it doesn't give enough control over cache flushing for smaller batches.
752 // Note that the import updates a lot of enities & checking & updating the group
753 // shouldn't add much performance wise. However, cache flushing will
754 $outcome = CRM_Core_BAO_EntityTag
::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE);
755 $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]];
757 return $tagAdditions;
761 * Determines the file extension based on error code.
763 * @var int $type error code constant
766 public static function errorFileName($type) {
772 $config = CRM_Core_Config
::singleton();
773 $fileName = $config->uploadDir
. "sqlImport";
776 $fileName .= '.errors';
779 case self
::DUPLICATE
:
780 $fileName .= '.duplicates';
784 $fileName .= '.mismatch';
787 case self
::UNPARSED_ADDRESS_WARNING
:
788 $fileName .= '.unparsedAddress';
796 * Determines the file name based on error code.
798 * @var int $type code constant
801 public static function saveFileName($type) {
808 $fileName = 'Import_Errors.csv';
811 case self
::DUPLICATE
:
812 $fileName = 'Import_Duplicates.csv';
816 $fileName = 'Import_Mismatch.csv';
819 case self
::UNPARSED_ADDRESS_WARNING
:
820 $fileName = 'Import_Unparsed_Address.csv';
828 * Check if contact is a duplicate .
830 * @param array $formatValues
834 protected function checkContactDuplicate(&$formatValues) {
835 //retrieve contact id using contact dedupe rule
836 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->getContactType();
837 $formatValues['version'] = 3;
838 require_once 'CRM/Utils/DeprecatedUtils.php';
839 $params = $formatValues;
840 static $cIndieFields = NULL;
841 static $defaultLocationId = NULL;
843 $contactType = $params['contact_type'];
844 if ($cIndieFields == NULL) {
845 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
846 $cIndieFields = $cTempIndieFields;
848 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
850 // set the value to default location id else set to 1
851 if (!$defaultLocationId = (int) $defaultLocation->id
) {
852 $defaultLocationId = 1;
856 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
858 $contactFormatted = [];
859 foreach ($params as $key => $field) {
860 if ($field == NULL ||
$field === '') {
863 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
864 // instead of soft credit contact.
865 if (is_array($field) && $key !== "soft_credit") {
866 foreach ($field as $value) {
868 if (is_array($value)) {
869 foreach ($value as $name => $testForEmpty) {
870 if ($name !== 'phone_type' &&
871 ($testForEmpty === '' ||
$testForEmpty == NULL)
882 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
888 $value = [$key => $field];
890 // check if location related field, then we need to add primary location type
891 if (in_array($key, $locationFields)) {
892 $value['location_type_id'] = $defaultLocationId;
894 elseif (array_key_exists($key, $cIndieFields)) {
895 $value['contact_type'] = $contactType;
898 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
901 $contactFormatted['contact_type'] = $contactType;
903 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
907 * This function adds the contact variable in $values to the
908 * parameter list $params. For most cases, $values should have length 1. If
909 * the variable being added is a child of Location, a location_type_id must
910 * also be included. If it is a child of phone, a phone_type must be included.
912 * @param array $values
913 * The variable(s) to be added.
914 * @param array $params
915 * The structured parameter list.
917 * @return bool|CRM_Utils_Error
919 private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
920 // @todo - like most functions in import ... most of this is cruft....
921 // Crawl through the possible classes:
934 // Cache the various object fields
935 static $fields = NULL;
937 if ($fields == NULL) {
941 // first add core contact values since for other Civi modules they are not added
942 require_once 'CRM/Contact/BAO/Contact.php';
943 $contactFields = CRM_Contact_DAO_Contact
::fields();
944 _civicrm_api3_store_values($contactFields, $values, $params);
946 if (isset($values['contact_type'])) {
947 // we're an individual/household/org property
949 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact
::fields();
951 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
955 if (isset($values['individual_prefix'])) {
956 if (!empty($params['prefix_id'])) {
957 $prefixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'prefix_id');
958 $params['prefix'] = $prefixes[$params['prefix_id']];
961 $params['prefix'] = $values['individual_prefix'];
966 if (isset($values['individual_suffix'])) {
967 if (!empty($params['suffix_id'])) {
968 $suffixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'suffix_id');
969 $params['suffix'] = $suffixes[$params['suffix_id']];
972 $params['suffix'] = $values['individual_suffix'];
977 if (isset($values['gender'])) {
978 if (!empty($params['gender_id'])) {
979 $genders = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'gender_id');
980 $params['gender'] = $genders[$params['gender_id']];
983 $params['gender'] = $values['gender'];
988 // format the website params.
989 if (!empty($values['url'])) {
990 static $websiteFields;
991 if (!is_array($websiteFields)) {
992 require_once 'CRM/Core/DAO/Website.php';
993 $websiteFields = CRM_Core_DAO_Website
::fields();
995 if (!array_key_exists('website', $params) ||
996 !is_array($params['website'])
998 $params['website'] = [];
1001 $websiteCount = count($params['website']);
1002 _civicrm_api3_store_values($websiteFields, $values,
1003 $params['website'][++
$websiteCount]
1009 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1010 if (!empty($values['location_type_id'])) {
1011 static $fields = NULL;
1012 if ($fields == NULL) {
1016 foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
1017 $name = strtolower($block);
1018 if (!array_key_exists($name, $values)) {
1022 if ($name === 'phone_ext') {
1026 // block present in value array.
1027 if (!array_key_exists($name, $params) ||
!is_array($params[$name])) {
1028 $params[$name] = [];
1031 if (!array_key_exists($block, $fields)) {
1032 $className = "CRM_Core_DAO_$block";
1033 $fields[$block] =& $className::fields();
1036 $blockCnt = count($params[$name]);
1038 // copy value to dao field name.
1039 if ($name == 'im') {
1040 $values['name'] = $values[$name];
1043 _civicrm_api3_store_values($fields[$block], $values,
1044 $params[$name][++
$blockCnt]
1047 if (empty($params['id']) && ($blockCnt == 1)) {
1048 $params[$name][$blockCnt]['is_primary'] = TRUE;
1051 // we only process single block at a time.
1055 // handle address fields.
1056 if (!array_key_exists('address', $params) ||
!is_array($params['address'])) {
1057 $params['address'] = [];
1061 foreach ($params['address'] as $cnt => $addressBlock) {
1062 if (CRM_Utils_Array
::value('location_type_id', $values) ==
1063 CRM_Utils_Array
::value('location_type_id', $addressBlock)
1071 if (!array_key_exists('Address', $fields)) {
1072 $fields['Address'] = CRM_Core_DAO_Address
::fields();
1075 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1076 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1077 // the address in CRM_Core_BAO_Address::create method
1078 if (!empty($values['location_type_id'])) {
1079 static $customFields = [];
1080 if (empty($customFields)) {
1081 $customFields = CRM_Core_BAO_CustomField
::getFields('Address');
1083 // make a copy of values, as we going to make changes
1084 $newValues = $values;
1085 foreach ($values as $key => $val) {
1086 $customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key);
1087 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1088 // mark an entry in fields array since we want the value of custom field to be copied
1089 $fields['Address'][$key] = NULL;
1091 $htmlType = $customFields[$customFieldID]['html_type'] ??
NULL;
1092 if (CRM_Core_BAO_CustomField
::isSerialized($customFields[$customFieldID]) && $val) {
1093 $mulValues = explode(',', $val);
1094 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1095 $newValues[$key] = [];
1096 foreach ($mulValues as $v1) {
1097 foreach ($customOption as $v2) {
1098 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1099 (strtolower($v2['value']) == strtolower(trim($v1)))
1101 if ($htmlType == 'CheckBox') {
1102 $newValues[$key][$v2['value']] = 1;
1105 $newValues[$key][] = $v2['value'];
1113 // consider new values
1114 $values = $newValues;
1117 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1123 'supplemental_address_1',
1124 'supplemental_address_2',
1125 'supplemental_address_3',
1126 'StateProvince.name',
1129 foreach ($addressFields as $field) {
1130 if (array_key_exists($field, $values)) {
1131 if (!array_key_exists('address', $params)) {
1132 $params['address'] = [];
1134 $params['address'][$addressCnt][$field] = $values[$field];
1138 if ($addressCnt == 1) {
1140 $params['address'][$addressCnt]['is_primary'] = TRUE;
1145 if (isset($values['note'])) {
1147 if (!isset($params['note'])) {
1148 $params['note'] = [];
1150 $noteBlock = count($params['note']) +
1;
1152 $params['note'][$noteBlock] = [];
1153 if (!isset($fields['Note'])) {
1154 $fields['Note'] = CRM_Core_DAO_Note
::fields();
1157 // get the current logged in civicrm user
1158 $session = CRM_Core_Session
::singleton();
1159 $userID = $session->get('userID');
1162 $values['contact_id'] = $userID;
1165 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1170 // Check for custom field values
1172 if (empty($fields['custom'])) {
1173 $fields['custom'] = &CRM_Core_BAO_CustomField
::getFields(CRM_Utils_Array
::value('contact_type', $values),
1174 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1178 foreach ($values as $key => $value) {
1179 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
1180 // check if it's a valid custom field id
1182 if (!array_key_exists($customFieldID, $fields['custom'])) {
1183 return civicrm_api3_create_error('Invalid custom field ID');
1186 $params[$key] = $value;
1193 * Parse a field which could be represented by a label or name value rather than the DB value.
1195 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1197 * but if not available then see if we have a label that can be converted to a name.
1199 * @param string|int|null $submittedValue
1200 * @param array $fieldSpec
1201 * Metadata for the field
1205 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1206 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1207 if (!isset($fieldSpec['bao'])) {
1208 return $submittedValue;
1210 /* @var \CRM_Core_DAO $bao */
1211 $bao = $fieldSpec['bao'];
1212 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1213 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1214 if (isset($nameOptions[$submittedValue])) {
1215 return $submittedValue;
1217 if (in_array($submittedValue, $nameOptions)) {
1218 return array_search($submittedValue, $nameOptions, TRUE);
1221 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1222 if (isset($labelOptions[$submittedValue])) {
1223 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1229 * This is code extracted from 4 places where this exact snippet was being duplicated.
1231 * FIXME: Extracting this was a first step, but there's also
1232 * 1. Inconsistency in the way other select options are handled.
1233 * Contribution adds handling for Select/Radio/Autocomplete
1234 * Participant/Activity only handles Select/Radio and misses Autocomplete
1235 * Membership is missing all of it
1236 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1238 * @param $customFieldID
1243 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1244 $mulValues = explode(',', $value);
1245 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1247 foreach ($mulValues as $v1) {
1248 foreach ($customOption as $customValueID => $customLabel) {
1249 $customValue = $customLabel['value'];
1250 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1251 (strtolower(trim($customValue)) == strtolower(trim($v1)))
1253 $values[] = $customValue;
1261 * Validate that the field requirements are met in the params.
1263 * @param array $requiredFields
1264 * @param array $params
1265 * An array of required fields (fieldName => label)
1266 * - note this follows the and / or array nesting we see in permission checks
1269 * 'email' => ts('Email'),
1270 * ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1272 * Means 'email' OR 'first_name AND 'last_name'.
1273 * @param string $prefixString
1275 * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1277 protected function validateRequiredFields(array $requiredFields, array $params, $prefixString = ''): void
{
1278 $missingFields = [];
1279 foreach ($requiredFields as $key => $required) {
1280 if (!is_array($required)) {
1281 $importParameter = $params[$key] ??
[];
1282 if (!is_array($importParameter)) {
1283 if (!empty($importParameter)) {
1288 foreach ($importParameter as $locationValues) {
1289 if (!empty($locationValues[$key])) {
1295 $missingFields[$key] = $required;
1298 foreach ($required as $field => $label) {
1299 if (empty($params[$field])) {
1300 $missing[$field] = $label;
1303 if (empty($missing)) {
1306 $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1309 throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1313 * Get the field value, transformed by metadata.
1315 * @param string $fieldName
1316 * @param string|int $importedValue
1317 * Value as it came in from the datasource.
1319 * @return string|array|bool|int
1320 * @throws \API_Exception
1322 protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1323 if (empty($importedValue)) {
1324 return $importedValue;
1326 $fieldMetadata = $this->getFieldMetadata($fieldName);
1327 if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) {
1329 foreach (explode(',', $importedValue) as $value) {
1330 $values[] = $this->getTransformedFieldValue($fieldName, trim($value));
1334 if ($fieldName === 'url') {
1335 return CRM_Utils_Rule
::url($importedValue) ?
$importedValue : 'invalid_import_value';
1338 if ($fieldName === 'email') {
1339 return CRM_Utils_Rule
::email($importedValue) ?
$importedValue : 'invalid_import_value';
1342 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_FLOAT
) {
1343 return CRM_Utils_Rule
::numeric($importedValue) ?
$importedValue : 'invalid_import_value';
1345 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_MONEY
) {
1346 return CRM_Utils_Rule
::money($importedValue, TRUE) ? CRM_Utils_Rule
::cleanMoney($importedValue) : 'invalid_import_value';
1348 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_BOOLEAN
) {
1349 $value = CRM_Utils_String
::strtoboolstr($importedValue);
1350 if ($value !== FALSE) {
1351 return (bool) $value;
1353 return 'invalid_import_value';
1355 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_DATE ||
$fieldMetadata['type'] === (CRM_Utils_Type
::T_DATE + CRM_Utils_Type
::T_TIME
) ||
$fieldMetadata['type'] === CRM_Utils_Type
::T_TIMESTAMP
) {
1356 $value = CRM_Utils_Date
::formatDate($importedValue, (int) $this->getSubmittedValue('dateFormats'));
1357 return $value ?
: 'invalid_import_value';
1359 $options = $this->getFieldOptions($fieldName);
1360 if ($options !== FALSE) {
1361 if ($this->isAmbiguous($fieldName, $importedValue)) {
1362 // We can't transform it at this stage. Perhaps later we can with
1363 // other information such as country.
1364 return $importedValue;
1367 $comparisonValue = $this->getComparisonValue($importedValue);
1368 return $options[$comparisonValue] ??
'invalid_import_value';
1370 if (!empty($fieldMetadata['FKClassName']) ||
!empty($fieldMetadata['pseudoconstant']['prefetch'])) {
1371 // @todo - make this generic - for fields where getOptions doesn't fetch
1372 // getOptions does not retrieve these fields with high potential results
1373 if ($fieldName === 'event_id') {
1374 if (!isset(Civi
::$statics[__CLASS__
][$fieldName][$importedValue])) {
1375 $event = Event
::get()->addClause('OR', ['title', '=', $importedValue], ['id', '=', $importedValue])->addSelect('id')->execute()->first();
1376 Civi
::$statics[__CLASS__
][$fieldName][$importedValue] = $event['id'] ??
FALSE;
1378 return Civi
::$statics[__CLASS__
][$fieldName][$importedValue] ??
'invalid_import_value';
1380 if ($fieldMetadata['name'] === 'campaign_id') {
1381 if (!isset(Civi
::$statics[__CLASS__
][$fieldName][$importedValue])) {
1382 $campaign = Campaign
::get()->addClause('OR', ['title', '=', $importedValue], ['name', '=', $importedValue])->addSelect('id')->execute()->first();
1383 Civi
::$statics[__CLASS__
][$fieldName][$importedValue] = $campaign['id'] ??
FALSE;
1385 return Civi
::$statics[__CLASS__
][$fieldName][$importedValue] ??
'invalid_import_value';
1388 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_INT
) {
1389 // We have resolved the options now so any remaining ones should be integers.
1390 return CRM_Utils_Rule
::numeric($importedValue) ?
$importedValue : 'invalid_import_value';
1392 return $importedValue;
1396 * @param string $fieldName
1398 * @return false|array
1400 * @throws \API_Exception
1402 protected function getFieldOptions(string $fieldName) {
1403 return $this->getFieldMetadata($fieldName, TRUE)['options'];
1407 * Get the metadata for the field.
1409 * @param string $fieldName
1410 * @param bool $loadOptions
1411 * @param bool $limitToContactType
1412 * Only show fields for the type to import (not appropriate when looking up
1413 * related contact fields).
1417 * @noinspection PhpDocMissingThrowsInspection
1418 * @noinspection PhpUnhandledExceptionInspection
1420 protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1422 $fieldMap = $this->getOddlyMappedMetadataFields();
1423 $fieldMapName = empty($fieldMap[$fieldName]) ?
$fieldName : $fieldMap[$fieldName];
1425 // This whole business of only loading metadata for one type when we actually need it for all is ... dubious.
1426 if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) {
1427 if ($loadOptions ||
!$limitToContactType) {
1428 $this->importableFieldsMetadata
[$fieldMapName] = CRM_Contact_BAO_Contact
::importableFields('All')[$fieldMapName];
1432 $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName];
1433 if ($loadOptions && !isset($fieldMetadata['options'])) {
1434 if (($fieldMetadata['data_type'] ??
'') === 'StateProvince') {
1435 // Probably already loaded and also supports abbreviations - eg. NSW.
1436 // Supporting for core AND custom state fields is more consistent.
1437 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id');
1438 return $this->importableFieldsMetadata
[$fieldMapName];
1440 if (($fieldMetadata['data_type'] ??
'') === 'Country') {
1441 // Probably already loaded and also supports abbreviations - eg. NSW.
1442 // Supporting for core AND custom state fields is more consistent.
1443 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $this->getFieldOptions('country_id');
1444 return $this->importableFieldsMetadata
[$fieldMapName];
1446 $optionFieldName = empty($fieldMap[$fieldName]) ?
$fieldMetadata['name'] : $fieldName;
1448 if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
1449 $options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
1450 'loadOptions' => ['id', 'name', 'label', 'abbr'],
1451 'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
1452 'select' => ['options'],
1453 ])->first()['options'];
1456 if (!empty($fieldMetadata['custom_group_id'])) {
1457 $customField = CustomField
::get(FALSE)
1458 ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
1459 ->addSelect('name', 'custom_group_id.name')
1462 $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
1464 $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1465 'loadOptions' => ['id', 'name', 'label', 'abbr'],
1466 'where' => [['name', '=', $optionFieldName]],
1467 'select' => ['options'],
1468 ])->first()['options'];
1470 if (is_array($options)) {
1471 // We create an array of the possible variants - notably including
1472 // name AND label as either might be used. We also lower case before checking
1474 foreach ($options as $option) {
1475 $idKey = $this->getComparisonValue($option['id']);
1476 $values[$idKey] = $option['id'];
1477 foreach (['name', 'label', 'abbr'] as $key) {
1478 $optionValue = $this->getComparisonValue($option[$key] ??
'');
1479 if ($optionValue !== '') {
1480 if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
1481 if (!isset($this->ambiguousOptions
[$fieldName][$optionValue])) {
1482 $this->ambiguousOptions
[$fieldName][$optionValue] = [$values[$optionValue]];
1484 $this->ambiguousOptions
[$fieldName][$optionValue][] = $option['id'];
1487 $values[$optionValue] = $option['id'];
1492 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $values;
1495 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $options ?
: FALSE;
1497 return $this->importableFieldsMetadata
[$fieldMapName];
1499 return $fieldMetadata;
1503 * Get the field metadata for fields to be be offered to match the contact.
1506 * @noinspection PhpDocMissingThrowsInspection
1508 protected function getContactMatchingFields(): array {
1509 $contactFields = CRM_Contact_BAO_Contact
::importableFields($this->getContactType(), NULL);
1510 $fields = ['external_identifier' => $contactFields['external_identifier']];
1511 $fields['external_identifier']['title'] .= ' (match to contact)';
1512 // Using new Dedupe rule.
1514 'contact_type' => $this->getContactType(),
1515 'used' => $this->getSubmittedValue('dedupe_rule_id') ??
'Unsupervised',
1517 $fieldsArray = CRM_Dedupe_BAO_DedupeRule
::dedupeRuleFields($ruleParams);
1519 if (is_array($fieldsArray)) {
1520 foreach ($fieldsArray as $value) {
1521 $customFieldId = CRM_Core_DAO
::getFieldValue('CRM_Core_DAO_CustomField',
1526 $value = trim($customFieldId ?
'custom_' . $customFieldId : $value);
1527 $fields[$value] = $contactFields[$value] ??
NULL;
1528 $title = $fields[$value]['title'] . ' (match to contact)';
1529 $fields[$value]['title'] = $title;
1536 * @param $customFieldID
1538 * @param array $fieldMetaData
1543 protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?
string {
1544 /* validate the data against the CF type */
1547 $dataType = $fieldMetaData['data_type'];
1548 $htmlType = $fieldMetaData['html_type'];
1549 $isSerialized = CRM_Core_BAO_CustomField
::isSerialized($fieldMetaData);
1550 if ($dataType === 'Date') {
1551 $params = ['date_field' => $value];
1552 if (CRM_Utils_Date
::convertToDefaultDate($params, $dateType, 'date_field')) {
1555 return $fieldMetaData['label'];
1557 elseif ($dataType === 'Boolean') {
1558 if (CRM_Utils_String
::strtoboolstr($value) === FALSE) {
1559 return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1562 // need not check for label filed import
1563 $selectHtmlTypes = [
1568 if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) ||
$dataType == 'Boolean' ||
$dataType == 'ContactReference') {
1569 $valid = CRM_Core_BAO_CustomValue
::typecheck($dataType, $value);
1571 return $fieldMetaData['label'];
1575 // check for values for custom fields for checkboxes and multiselect
1576 if ($isSerialized && $dataType != 'ContactReference') {
1577 $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1578 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1579 foreach ($mulValues as $v1) {
1582 foreach ($customOption as $v2) {
1583 if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) ||
(strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1589 return $fieldMetaData['label'];
1593 elseif ($htmlType == 'Select' ||
($htmlType == 'Radio' && $dataType != 'Boolean')) {
1594 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1596 foreach ($customOption as $v2) {
1597 if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) ||
(strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1602 return $fieldMetaData['label'];
1611 * Get the entity for the given field.
1613 * @param string $fieldName
1615 * @return mixed|null
1616 * @throws \API_Exception
1618 protected function getFieldEntity(string $fieldName) {
1619 if ($fieldName === 'do_not_import') {
1622 if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) {
1625 $metadata = $this->getFieldMetadata($fieldName);
1626 if (!isset($metadata['entity'])) {
1627 return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ?
'Contact' : $metadata['extends'];
1630 // Our metadata for these is fugly. Handling the fugliness during retrieval.
1631 if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1634 return $metadata['entity'];
1638 * Validate the import file, updating the import table with results.
1640 * @throws \API_Exception
1641 * @throws \CRM_Core_Exception
1643 public function validate(): void
{
1644 $dataSource = $this->getDataSourceObject();
1645 while ($row = $dataSource->getRow()) {
1647 $rowNumber = $row['_id'];
1648 $values = array_values($row);
1649 $this->validateValues($values);
1650 $this->setImportStatus($rowNumber, 'NEW', '');
1652 catch (CRM_Core_Exception
$e) {
1653 $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
1659 * Validate the import values.
1661 * The values array represents a row in the datasource.
1663 * @param array $values
1665 * @throws \API_Exception
1666 * @throws \CRM_Core_Exception
1668 public function validateValues(array $values): void
{
1669 $params = $this->getMappedRow($values);
1670 $this->validateParams($params);
1674 * @param array $params
1676 * @throws \CRM_Core_Exception
1678 protected function validateParams(array $params): void
{
1679 if (empty($params['id'])) {
1680 $this->validateRequiredFields($this->getRequiredFields(), $params);
1683 foreach ($params as $key => $value) {
1684 $errors = array_merge($this->getInvalidValues($value, $key), $errors);
1687 throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors));
1692 * Search the value for the string 'invalid_import_value'.
1694 * If the string is found it indicates the fields was rejected
1695 * during `getTransformedValue` as not having valid data.
1697 * @param string|array|int $value
1698 * @param string $key
1699 * @param string $prefixString
1703 protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
1705 if ($value === 'invalid_import_value') {
1706 if (!is_numeric($key)) {
1707 $metadata = $this->getFieldMetadata($key);
1708 $errors[] = $prefixString . ($metadata['html']['label'] ??
$metadata['title']);
1711 // Numeric key suggests we are drilling into option values
1715 elseif (is_array($value)) {
1716 foreach ($value as $innerKey => $innerValue) {
1717 $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1718 if ($result === [TRUE]) {
1719 $metadata = $this->getFieldMetadata($key);
1720 $errors[] = $prefixString . ($metadata['html']['label'] ??
$metadata['title']);
1722 elseif (!empty($result)) {
1723 $errors = array_merge($result, $errors);
1727 return array_filter($errors);
1731 * Get the available countries.
1733 * If the site is not configured with a restriction then all countries are valid
1734 * but otherwise only a select array are.
1736 * @return array|false
1737 * FALSE indicates no restrictions.
1739 protected function getAvailableCountries() {
1740 if ($this->availableCountries
=== NULL) {
1741 $availableCountries = Civi
::settings()->get('countryLimit');
1742 $this->availableCountries
= !empty($availableCountries) ?
array_fill_keys($availableCountries, TRUE) : FALSE;
1744 return $this->availableCountries
;
1748 * Get the metadata field for which importable fields does not key the actual field name.
1752 protected function getOddlyMappedMetadataFields(): array {
1754 'country_id' => 'country',
1755 'state_province_id' => 'state_province',
1756 'county_id' => 'county',
1757 'email_greeting_id' => 'email_greeting',
1758 'postal_greeting_id' => 'postal_greeting',
1759 'addressee_id' => 'addressee',
1764 * Get the default country for the site.
1768 protected function getSiteDefaultCountry(): int {
1769 if (!isset($this->siteDefaultCountry
)) {
1770 $this->siteDefaultCountry
= (int) Civi
::settings()->get('defaultContactCountry');
1772 return $this->siteDefaultCountry
;
1776 * Is the option ambiguous.
1778 * @param string $fieldName
1779 * @param string $importedValue
1781 protected function isAmbiguous(string $fieldName, $importedValue): bool {
1782 return !empty($this->ambiguousOptions
[$fieldName][$this->getComparisonValue($importedValue)]);
1786 * Get the civicrm_mapping_field appropriate layout for the mapper input.
1788 * For simple parsers (not contribution or contact) the input looks like
1789 * ['first_name', 'custom_32']
1790 * and it is converted to
1792 * ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5],
1794 * @param array $fieldMapping
1795 * @param int $mappingID
1796 * @param int $columnNumber
1800 public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array {
1802 'name' => $fieldMapping[0],
1803 'mapping_id' => $mappingID,
1804 'column_number' => $columnNumber,
1809 * The initializer code, called before the processing
1813 public function init() {
1814 // Force re-load of user job.
1815 unset($this->userJob
);
1816 $this->setFieldMetadata();
1820 * @param array $mappedField
1821 * Field detail as would be saved in field_mapping table
1822 * or as returned from getMappingFieldFromMapperInput
1825 * @throws \API_Exception
1827 public function getMappedFieldLabel(array $mappedField): string {
1828 // doNotImport is on it's way out - skip fields will be '' once all is done.
1829 if ($mappedField['name'] === 'doNotImport') {
1832 $this->setFieldMetadata();
1833 $metadata = $this->getFieldMetadata($mappedField['name']);
1834 return $metadata['html']['label'] ??
$metadata['title'];
1838 * Get the row from the csv mapped to our parameters.
1840 * @param array $values
1843 * @throws \API_Exception
1845 public function getMappedRow(array $values): array {
1847 foreach ($this->getFieldMappings() as $i => $mappedField) {
1848 if ($mappedField['name'] === 'do_not_import') {
1851 if ($mappedField['name']) {
1852 $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
1859 * Get the field mappings for the import.
1861 * This is the same format as saved in civicrm_mapping_field except
1862 * that location_type_id = 'Primary' rather than empty where relevant.
1863 * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
1866 * @throws \API_Exception
1868 protected function getFieldMappings(): array {
1870 $mapper = $this->getSubmittedValue('mapper');
1871 foreach ($mapper as $i => $mapperRow) {
1872 $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
1873 // Just for clarity since 0 is a pseudo-value
1874 unset($mappedField['mapping_id']);
1875 $mappedFields[] = $mappedField;
1877 return $mappedFields;
1883 * @param \CRM_Queue_TaskContext $taskContext
1885 * @param int $userJobID
1889 * @throws \API_Exception
1890 * @throws \CRM_Core_Exception
1892 public static function runImport($taskContext, $userJobID, $limit) {
1893 $userJob = UserJob
::get()->addWhere('id', '=', $userJobID)->addSelect('job_type')->execute()->first();
1894 $parserClass = NULL;
1895 foreach (CRM_Core_BAO_UserJob
::getTypes() as $userJobType) {
1896 if ($userJob['job_type'] === $userJobType['id']) {
1897 $parserClass = $userJobType['class'];
1900 /* @var \CRM_Import_Parser $parser */
1901 $parser = new $parserClass();
1902 $parser->setUserJobID($userJobID);
1903 // Not sure if we still need to init....
1905 $dataSource = $parser->getDataSourceObject();
1906 $dataSource->setStatuses(['new']);
1907 $dataSource->setLimit($limit);
1909 while ($row = $dataSource->getRow()) {
1910 $values = array_values($row);
1911 $parser->import($values);
1913 $parser->doPostImportActions();
1918 * Check if an error in custom data.
1920 * @deprecated all of this is duplicated if getTransformedValue is used.
1922 * @param array $params
1923 * @param string $errorMessage
1924 * A string containing all the error-fields.
1926 * @param null $csType
1928 public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) {
1929 $dateType = CRM_Core_Session
::singleton()->get("dateTypes");
1932 if (!empty($params['contact_sub_type'])) {
1933 $csType = $params['contact_sub_type'] ??
NULL;
1936 if (empty($params['contact_type'])) {
1937 $params['contact_type'] = 'Individual';
1940 // get array of subtypes - CRM-18708
1941 if (in_array($csType, CRM_Contact_BAO_ContactType
::basicTypes(TRUE), TRUE)) {
1942 $csType = $this->getSubtypes($params['contact_type']);
1945 if (is_array($csType)) {
1946 // fetch custom fields for every subtype and add it to $customFields array
1949 foreach ($csType as $cType) {
1950 $customFields +
= CRM_Core_BAO_CustomField
::getFields($params['contact_type'], FALSE, FALSE, $cType);
1954 $customFields = CRM_Core_BAO_CustomField
::getFields($params['contact_type'], FALSE, FALSE, $csType);
1957 foreach ($params as $key => $value) {
1958 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
1959 //For address custom fields, we do get actual custom field value as an inner array of
1960 //values so need to modify
1961 if (!array_key_exists($customFieldID, $customFields)) {
1962 return ts('field ID');
1964 /* check if it's a valid custom field id */
1965 $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType);
1969 $errorMessage .= ($errorMessage ?
'; ' : '') . implode('; ', array_filter($errors));
1974 * get subtypes given the contact type
1976 * @param string $contactType
1977 * @return array $subTypes
1979 protected function getSubtypes($contactType) {
1981 $types = CRM_Contact_BAO_ContactType
::subTypeInfo($contactType);
1983 if (count($types) > 0) {
1984 foreach ($types as $type) {
1985 $subTypes[] = $type['name'];
1992 * Update the status of the import row to reflect the processing outcome.
1995 * @param string $status
1996 * @param string $message
1997 * @param int|null $entityID
1998 * Optional created entity ID
1999 * @param array $additionalFields
2000 * Additional fields to be tracked
2001 * @param array $createdContactIDs
2003 * @noinspection PhpDocMissingThrowsInspection
2004 * @noinspection PhpUnhandledExceptionInspection
2006 protected function setImportStatus(int $id, string $status, string $message = '', ?
int $entityID = NULL, $additionalFields = [], $createdContactIDs = []): void
{
2007 foreach ($createdContactIDs as $createdContactID) {
2008 // Store any created contacts for post_actions like tag or add to group.
2009 // These are done on a 'per-batch' status in processPorstActions
2010 // so holding in a property is OK.
2011 $this->createdContacts
[$createdContactID] = $createdContactID;
2013 $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields);
2017 * Convert any given date string to default date array.
2019 * @param array $params
2020 * Has given date-format.
2021 * @param array $formatted
2022 * Store formatted date in this array.
2023 * @param int $dateType
2025 * @param string $dateParam
2028 public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) {
2030 CRM_Utils_Date
::convertToDefaultDate($params, $dateType, $dateParam);
2031 $formatted[$dateParam] = CRM_Utils_Date
::processDate($params[$dateParam]);
2035 * Get the value to use for option comparison purposes.
2037 * We do a case-insensitive comparison, also swapping ’ for '
2038 * which has at least one known usage (Côte d’Ivoire).
2040 * Note we do this to both sides of the comparison.
2042 * @param int|string|false|null $importedValue
2044 * @return false|int|string|null
2046 protected function getComparisonValue($importedValue) {
2047 return is_numeric($importedValue) ?
$importedValue : mb_strtolower(str_replace('’', "'", $importedValue));