3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
12 use Civi\Api4\UserJob
;
17 * @copyright CiviCRM LLC https://civicrm.org/licensing
19 abstract class CRM_Import_Parser
{
23 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
28 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
33 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
36 * Codes for duplicate record handling
38 const DUPLICATE_SKIP
= 1, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
43 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
48 * This is the primary key of the civicrm_user_job table which is used to
56 * Fields which are being handled by metadata formatting & validation functions.
58 * This is intended as a temporary parameter as we phase in metadata handling.
60 * The end result is that all fields will be & this will go but for now it is
65 protected $metadataHandledFields = [];
70 public function getUserJobID(): ?
int {
71 return $this->userJobID
;
77 * @param int $userJobID
81 public function setUserJobID(int $userJobID): self
{
82 $this->userJobID
= $userJobID;
87 * Countries that the site is restricted to
91 private $availableCountries;
96 * API call to retrieve the userJob row.
100 * @throws \API_Exception
102 protected function getUserJob(): array {
103 return UserJob
::get()
104 ->addWhere('id', '=', $this->getUserJobID())
110 * Get the relevant datasource object.
112 * @return \CRM_Import_DataSource|null
114 * @throws \API_Exception
116 protected function getDataSourceObject(): ?CRM_Import_DataSource
{
117 $className = $this->getSubmittedValue('dataSource');
119 /* @var CRM_Import_DataSource $dataSource */
120 return new $className($this->getUserJobID());
126 * Get the submitted value, as stored on the user job.
128 * @param string $fieldName
132 * @throws \API_Exception
134 protected function getSubmittedValue(string $fieldName) {
135 return $this->getUserJob()['metadata']['submitted_values'][$fieldName];
139 * Has the import completed.
143 * @throws \API_Exception
144 * @throws \CRM_Core_Exception
146 public function isComplete() :bool {
147 return $this->getDataSourceObject()->isCompleted();
151 * Get configured contact type.
153 * @throws \API_Exception
155 protected function getContactType() {
156 if (!$this->_contactType
) {
157 $contactTypeMapping = [
158 CRM_Import_Parser
::CONTACT_INDIVIDUAL
=> 'Individual',
159 CRM_Import_Parser
::CONTACT_HOUSEHOLD
=> 'Household',
160 CRM_Import_Parser
::CONTACT_ORGANIZATION
=> 'Organization',
162 $this->_contactType
= $contactTypeMapping[$this->getSubmittedValue('contactType')];
164 return $this->_contactType
;
168 * Get configured contact type.
170 * @return string|null
172 * @throws \API_Exception
174 public function getContactSubType() {
175 if (!$this->_contactSubType
) {
176 $this->_contactSubType
= $this->getSubmittedValue('contactSubType');
178 return $this->_contactSubType
;
182 * Total number of non empty lines
185 protected $_totalCount;
188 * Running total number of valid lines
191 protected $_validCount;
194 * Running total number of invalid rows
197 protected $_invalidRowCount;
200 * Maximum number of non-empty/comment lines to process
204 protected $_maxLinesToProcess;
207 * Array of error lines, bounded by MAX_ERROR
213 * Total number of duplicate (from database) lines
216 protected $_duplicateCount;
219 * Array of duplicate lines
222 protected $_duplicates;
225 * Maximum number of warnings to store
228 protected $_maxWarningCount = self
::MAX_WARNINGS
;
231 * Array of warning lines, bounded by MAX_WARNING
234 protected $_warnings;
237 * Array of all the fields that could potentially be part
238 * of this import process
244 * Metadata for all available fields, keyed by unique name.
246 * This is intended to supercede $_fields which uses a special sauce format which
247 * importableFieldsMetadata uses the standard getfields type format.
251 protected $importableFieldsMetadata = [];
254 * Get metadata for all importable fields in std getfields style format.
258 public function getImportableFieldsMetadata(): array {
259 return $this->importableFieldsMetadata
;
263 * Set metadata for all importable fields in std getfields style format.
265 * @param array $importableFieldsMetadata
267 public function setImportableFieldsMetadata(array $importableFieldsMetadata): void
{
268 $this->importableFieldsMetadata
= $importableFieldsMetadata;
272 * Gets the fields available for importing in a key-name, title format.
275 * eg. ['first_name' => 'First Name'.....]
277 * @throws \API_Exception
279 * @todo - we are constructing the metadata before we
280 * have set the contact type so we re-do it here.
282 * Once we have cleaned up the way the mapper is handled
283 * we can ditch all the existing _construct parameters in favour
284 * of just the userJobID - there are current open PRs towards this end.
286 public function getAvailableFields(): array {
287 $this->setFieldMetadata();
289 foreach ($this->getImportableFieldsMetadata() as $name => $field) {
290 if ($name === 'id' && $this->isSkipDuplicates()) {
291 // Duplicates are being skipped so id matching is not availble.
294 $return[$name] = $field['html']['label'] ??
$field['title'];
300 * Did the user specify duplicates should be skipped and not imported.
304 * @throws \API_Exception
306 protected function isSkipDuplicates(): bool {
307 return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser
::DUPLICATE_SKIP
;
311 * Array of the fields that are actually part of the import process
312 * the position in the array also dictates their position in the import
316 protected $_activeFields;
319 * Cache the count of active fields
323 protected $_activeFieldCount;
326 * Cache of preview rows
333 * Filename of error data
337 protected $_errorFileName;
340 * Filename of duplicate data
344 protected $_duplicateFileName;
351 public $_contactType;
354 * @param string $contactType
356 * @return CRM_Import_Parser
358 public function setContactType(string $contactType): CRM_Import_Parser
{
359 $this->_contactType
= $contactType;
368 public $_contactSubType;
371 * @param int|null $contactSubType
375 public function setContactSubType(?
int $contactSubType): self
{
376 $this->_contactSubType
= $contactSubType;
383 public function __construct() {
384 $this->_maxLinesToProcess
= 0;
388 * Set and validate field values.
390 * @param array $elements
393 public function setActiveFieldValues($elements): void
{
394 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
395 for ($i = 0; $i < $maxCount; $i++
) {
396 $this->_activeFields
[$i]->setValue($elements[$i]);
399 // reset all the values that we did not have an equivalent import element
400 for (; $i < $this->_activeFieldCount
; $i++
) {
401 $this->_activeFields
[$i]->resetValue();
406 * Format the field values for input to the api.
409 * (reference) associative array of name/value pairs
411 public function &getActiveFieldParams() {
413 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
414 if (isset($this->_activeFields
[$i]->_value
)
415 && !isset($params[$this->_activeFields
[$i]->_name
])
416 && !isset($this->_activeFields
[$i]->_related
)
419 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
426 * Add progress bar to the import process. Calculates time remaining, status etc.
429 * status id of the import process saved in $config->uploadDir.
430 * @param bool $startImport
431 * True when progress bar is to be initiated.
432 * @param $startTimestamp
433 * Initial timestamp when the import was started.
434 * @param $prevTimestamp
435 * Previous timestamp when this function was last called.
436 * @param $totalRowCount
437 * Total number of rows in the import file.
439 * @return NULL|$currTimestamp
441 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
442 $statusFile = CRM_Core_Config
::singleton()->uploadDir
. "status_{$statusID}.txt";
445 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
446 //do not force the browser to display the save dialog, CRM-7640
447 $contents = json_encode([0, $status]);
448 file_put_contents($statusFile, $contents);
451 $rowCount = $this->_rowCount ??
$this->_lineCount
;
452 $currTimestamp = time();
453 $time = ($currTimestamp - $prevTimestamp);
454 $recordsLeft = $totalRowCount - $rowCount;
455 if ($recordsLeft < 0) {
458 $estimatedTime = ($recordsLeft / 50) * $time;
459 $estMinutes = floor($estimatedTime / 60);
461 if ($estMinutes > 1) {
462 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
463 $estimatedTime = $estimatedTime - ($estMinutes * 60);
465 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
466 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
467 $statusMsg = ts('%1 of %2 records - %3 remaining',
468 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
470 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
471 $contents = json_encode([$processedPercent, $status]);
473 file_put_contents($statusFile, $contents);
474 return $currTimestamp;
481 public function getSelectValues(): array {
483 foreach ($this->_fields
as $name => $field) {
484 $values[$name] = $field->_title
;
492 public function getSelectTypes() {
494 // This is only called from the MapField form in isolation now,
495 // so we need to set the metadata.
497 foreach ($this->_fields
as $name => $field) {
498 if (isset($field->_hasLocationType
)) {
499 $values[$name] = $field->_hasLocationType
;
508 public function getHeaderPatterns() {
510 foreach ($this->_fields
as $name => $field) {
511 if (isset($field->_headerPattern
)) {
512 $values[$name] = $field->_headerPattern
;
521 public function getDataPatterns() {
523 foreach ($this->_fields
as $name => $field) {
524 $values[$name] = $field->_dataPattern
;
530 * Remove single-quote enclosures from a value array (row).
532 * @param array $values
533 * @param string $enclosure
537 public static function encloseScrub(&$values, $enclosure = "'") {
538 if (empty($values)) {
542 foreach ($values as $k => $v) {
543 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
554 public function setMaxLinesToProcess($max) {
555 $this->_maxLinesToProcess
= $max;
559 * Validate that we have the required fields to create the contact or find it to update.
561 * Note that the users duplicate selection affects this as follows
562 * - if they did not select an update variant then the id field is not
563 * permitted in the mapping - so we can assume the presence of id means
565 * - the external_identifier field is valid in place of the other fields
566 * when they have chosen update or fill - in this case we are only looking
567 * to update an existing contact.
569 * @param string $contactType
570 * @param array $params
571 * @param bool $isPermitExistingMatchFields
572 * True if the it is enough to have fields which will enable us to find
573 * an existing contact (eg. external_identifier).
574 * @param string $prefixString
575 * String to include in the exception (e.g '(Child of)' if we are validating
579 * @throws \CRM_Core_Exception
581 protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void
{
582 if (!empty($params['id'])) {
587 'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
588 'email' => ts('Email Address'),
590 'Organization' => ['organization_name' => ts('Organization Name')],
591 'Household' => ['household_name' => ts('Household Name')],
593 if ($isPermitExistingMatchFields) {
594 $requiredFields['external_identifier'] = ts('External Identifier');
595 // Historically just an email has been accepted as it is 'usually good enough'
596 // for a dedupe rule look up - but really this is a stand in for
597 // whatever is needed to find an existing matching contact using the
598 // specified dedupe rule (or the default Unsupervised if not specified).
599 $requiredFields['email'] = ts('Email Address');
601 $this->validateRequiredFields($requiredFields, $params, $prefixString);
605 * Determines the file extension based on error code.
607 * @var int $type error code constant
610 public static function errorFileName($type) {
616 $config = CRM_Core_Config
::singleton();
617 $fileName = $config->uploadDir
. "sqlImport";
620 $fileName .= '.errors';
623 case self
::DUPLICATE
:
624 $fileName .= '.duplicates';
628 $fileName .= '.mismatch';
631 case self
::UNPARSED_ADDRESS_WARNING
:
632 $fileName .= '.unparsedAddress';
640 * Determines the file name based on error code.
642 * @var $type error code constant
645 public static function saveFileName($type) {
652 $fileName = 'Import_Errors.csv';
655 case self
::DUPLICATE
:
656 $fileName = 'Import_Duplicates.csv';
660 $fileName = 'Import_Mismatch.csv';
663 case self
::UNPARSED_ADDRESS_WARNING
:
664 $fileName = 'Import_Unparsed_Address.csv';
672 * Check if contact is a duplicate .
674 * @param array $formatValues
678 protected function checkContactDuplicate(&$formatValues) {
679 //retrieve contact id using contact dedupe rule
680 $formatValues['contact_type'] = $formatValues['contact_type'] ??
$this->_contactType
;
681 $formatValues['version'] = 3;
682 require_once 'CRM/Utils/DeprecatedUtils.php';
683 $params = $formatValues;
684 static $cIndieFields = NULL;
685 static $defaultLocationId = NULL;
687 $contactType = $params['contact_type'];
688 if ($cIndieFields == NULL) {
689 $cTempIndieFields = CRM_Contact_BAO_Contact
::importableFields($contactType);
690 $cIndieFields = $cTempIndieFields;
692 $defaultLocation = CRM_Core_BAO_LocationType
::getDefault();
694 // set the value to default location id else set to 1
695 if (!$defaultLocationId = (int) $defaultLocation->id
) {
696 $defaultLocationId = 1;
700 $locationFields = CRM_Contact_BAO_Query
::$_locationSpecificFields;
702 $contactFormatted = [];
703 foreach ($params as $key => $field) {
704 if ($field == NULL ||
$field === '') {
707 // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact
708 // instead of soft credit contact.
709 if (is_array($field) && $key != "soft_credit") {
710 foreach ($field as $value) {
712 if (is_array($value)) {
713 foreach ($value as $name => $testForEmpty) {
714 if ($name !== 'phone_type' &&
715 ($testForEmpty === '' ||
$testForEmpty == NULL)
726 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
732 $value = [$key => $field];
734 // check if location related field, then we need to add primary location type
735 if (in_array($key, $locationFields)) {
736 $value['location_type_id'] = $defaultLocationId;
738 elseif (array_key_exists($key, $cIndieFields)) {
739 $value['contact_type'] = $contactType;
742 $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted);
745 $contactFormatted['contact_type'] = $contactType;
747 return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted);
751 * This function adds the contact variable in $values to the
752 * parameter list $params. For most cases, $values should have length 1. If
753 * the variable being added is a child of Location, a location_type_id must
754 * also be included. If it is a child of phone, a phone_type must be included.
756 * @param array $values
757 * The variable(s) to be added.
758 * @param array $params
759 * The structured parameter list.
761 * @return bool|CRM_Utils_Error
763 private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) {
764 // @todo - like most functions in import ... most of this is cruft....
765 // Crawl through the possible classes:
778 // Cache the various object fields
779 static $fields = NULL;
781 if ($fields == NULL) {
785 // first add core contact values since for other Civi modules they are not added
786 require_once 'CRM/Contact/BAO/Contact.php';
787 $contactFields = CRM_Contact_DAO_Contact
::fields();
788 _civicrm_api3_store_values($contactFields, $values, $params);
790 if (isset($values['contact_type'])) {
791 // we're an individual/household/org property
793 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact
::fields();
795 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
799 if (isset($values['individual_prefix'])) {
800 if (!empty($params['prefix_id'])) {
801 $prefixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'prefix_id');
802 $params['prefix'] = $prefixes[$params['prefix_id']];
805 $params['prefix'] = $values['individual_prefix'];
810 if (isset($values['individual_suffix'])) {
811 if (!empty($params['suffix_id'])) {
812 $suffixes = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'suffix_id');
813 $params['suffix'] = $suffixes[$params['suffix_id']];
816 $params['suffix'] = $values['individual_suffix'];
822 if (isset($values['email_greeting'])) {
823 if (!empty($params['email_greeting_id'])) {
824 $emailGreetingFilter = [
825 'contact_type' => $params['contact_type'] ??
NULL,
826 'greeting_type' => 'email_greeting',
828 $emailGreetings = CRM_Core_PseudoConstant
::greeting($emailGreetingFilter);
829 $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
832 $params['email_greeting'] = $values['email_greeting'];
838 if (isset($values['postal_greeting'])) {
839 if (!empty($params['postal_greeting_id'])) {
840 $postalGreetingFilter = [
841 'contact_type' => $params['contact_type'] ??
NULL,
842 'greeting_type' => 'postal_greeting',
844 $postalGreetings = CRM_Core_PseudoConstant
::greeting($postalGreetingFilter);
845 $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
848 $params['postal_greeting'] = $values['postal_greeting'];
853 if (isset($values['addressee'])) {
854 $params['addressee'] = $values['addressee'];
858 if (isset($values['gender'])) {
859 if (!empty($params['gender_id'])) {
860 $genders = CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'gender_id');
861 $params['gender'] = $genders[$params['gender_id']];
864 $params['gender'] = $values['gender'];
869 if (!empty($values['preferred_communication_method'])) {
871 $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant
::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER
);
873 $preffComm = explode(',', $values['preferred_communication_method']);
874 foreach ($preffComm as $v) {
875 $v = strtolower(trim($v));
876 if (array_key_exists($v, $pcm)) {
881 $params['preferred_communication_method'] = $comm;
885 // format the website params.
886 if (!empty($values['url'])) {
887 static $websiteFields;
888 if (!is_array($websiteFields)) {
889 require_once 'CRM/Core/DAO/Website.php';
890 $websiteFields = CRM_Core_DAO_Website
::fields();
892 if (!array_key_exists('website', $params) ||
893 !is_array($params['website'])
895 $params['website'] = [];
898 $websiteCount = count($params['website']);
899 _civicrm_api3_store_values($websiteFields, $values,
900 $params['website'][++
$websiteCount]
906 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
907 if (!empty($values['location_type_id'])) {
908 static $fields = NULL;
909 if ($fields == NULL) {
913 foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) {
914 $name = strtolower($block);
915 if (!array_key_exists($name, $values)) {
919 if ($name === 'phone_ext') {
923 // block present in value array.
924 if (!array_key_exists($name, $params) ||
!is_array($params[$name])) {
928 if (!array_key_exists($block, $fields)) {
929 $className = "CRM_Core_DAO_$block";
930 $fields[$block] =& $className::fields();
933 $blockCnt = count($params[$name]);
935 // copy value to dao field name.
937 $values['name'] = $values[$name];
940 _civicrm_api3_store_values($fields[$block], $values,
941 $params[$name][++
$blockCnt]
944 if (empty($params['id']) && ($blockCnt == 1)) {
945 $params[$name][$blockCnt]['is_primary'] = TRUE;
948 // we only process single block at a time.
952 // handle address fields.
953 if (!array_key_exists('address', $params) ||
!is_array($params['address'])) {
954 $params['address'] = [];
958 foreach ($params['address'] as $cnt => $addressBlock) {
959 if (CRM_Utils_Array
::value('location_type_id', $values) ==
960 CRM_Utils_Array
::value('location_type_id', $addressBlock)
968 if (!array_key_exists('Address', $fields)) {
969 $fields['Address'] = CRM_Core_DAO_Address
::fields();
972 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
973 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
974 // the address in CRM_Core_BAO_Address::create method
975 if (!empty($values['location_type_id'])) {
976 static $customFields = [];
977 if (empty($customFields)) {
978 $customFields = CRM_Core_BAO_CustomField
::getFields('Address');
980 // make a copy of values, as we going to make changes
981 $newValues = $values;
982 foreach ($values as $key => $val) {
983 $customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key);
984 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
985 // mark an entry in fields array since we want the value of custom field to be copied
986 $fields['Address'][$key] = NULL;
988 $htmlType = $customFields[$customFieldID]['html_type'] ??
NULL;
989 if (CRM_Core_BAO_CustomField
::isSerialized($customFields[$customFieldID]) && $val) {
990 $mulValues = explode(',', $val);
991 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
992 $newValues[$key] = [];
993 foreach ($mulValues as $v1) {
994 foreach ($customOption as $v2) {
995 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
996 (strtolower($v2['value']) == strtolower(trim($v1)))
998 if ($htmlType == 'CheckBox') {
999 $newValues[$key][$v2['value']] = 1;
1002 $newValues[$key][] = $v2['value'];
1010 // consider new values
1011 $values = $newValues;
1014 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]);
1020 'supplemental_address_1',
1021 'supplemental_address_2',
1022 'supplemental_address_3',
1023 'StateProvince.name',
1026 foreach ($addressFields as $field) {
1027 if (array_key_exists($field, $values)) {
1028 if (!array_key_exists('address', $params)) {
1029 $params['address'] = [];
1031 $params['address'][$addressCnt][$field] = $values[$field];
1035 if ($addressCnt == 1) {
1037 $params['address'][$addressCnt]['is_primary'] = TRUE;
1042 if (isset($values['note'])) {
1044 if (!isset($params['note'])) {
1045 $params['note'] = [];
1047 $noteBlock = count($params['note']) +
1;
1049 $params['note'][$noteBlock] = [];
1050 if (!isset($fields['Note'])) {
1051 $fields['Note'] = CRM_Core_DAO_Note
::fields();
1054 // get the current logged in civicrm user
1055 $session = CRM_Core_Session
::singleton();
1056 $userID = $session->get('userID');
1059 $values['contact_id'] = $userID;
1062 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1067 // Check for custom field values
1069 if (empty($fields['custom'])) {
1070 $fields['custom'] = &CRM_Core_BAO_CustomField
::getFields(CRM_Utils_Array
::value('contact_type', $values),
1071 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1075 foreach ($values as $key => $value) {
1076 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
1077 // check if it's a valid custom field id
1079 if (!array_key_exists($customFieldID, $fields['custom'])) {
1080 return civicrm_api3_create_error('Invalid custom field ID');
1083 $params[$key] = $value;
1090 * Parse a field which could be represented by a label or name value rather than the DB value.
1092 * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id.
1094 * but if not available then see if we have a label that can be converted to a name.
1096 * @param string|int|null $submittedValue
1097 * @param array $fieldSpec
1098 * Metadata for the field
1102 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
1103 // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue
1104 if (!isset($fieldSpec['bao'])) {
1105 return $submittedValue;
1107 /* @var \CRM_Core_DAO $bao */
1108 $bao = $fieldSpec['bao'];
1109 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
1110 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
1111 if (isset($nameOptions[$submittedValue])) {
1112 return $submittedValue;
1114 if (in_array($submittedValue, $nameOptions)) {
1115 return array_search($submittedValue, $nameOptions, TRUE);
1118 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
1119 if (isset($labelOptions[$submittedValue])) {
1120 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);
1126 * This is code extracted from 4 places where this exact snippet was being duplicated.
1128 * FIXME: Extracting this was a first step, but there's also
1129 * 1. Inconsistency in the way other select options are handled.
1130 * Contribution adds handling for Select/Radio/Autocomplete
1131 * Participant/Activity only handles Select/Radio and misses Autocomplete
1132 * Membership is missing all of it
1133 * 2. Inconsistency with the way this works vs. how it's implemented in Contact import.
1135 * @param $customFieldID
1140 public static function unserializeCustomValue($customFieldID, $value, $fieldType) {
1141 $mulValues = explode(',', $value);
1142 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1144 foreach ($mulValues as $v1) {
1145 foreach ($customOption as $customValueID => $customLabel) {
1146 $customValue = $customLabel['value'];
1147 if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) ||
1148 (strtolower(trim($customValue)) == strtolower(trim($v1)))
1150 $values[] = $customValue;
1158 * Validate that the field requirements are met in the params.
1160 * @param array $requiredFields
1161 * @param array $params
1162 * An array of required fields (fieldName => label)
1163 * - note this follows the and / or array nesting we see in permission checks
1166 * 'email' => ts('Email'),
1167 * ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
1169 * Means 'email' OR 'first_name AND 'last_name'.
1170 * @param string $prefixString
1172 * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
1174 protected function validateRequiredFields(array $requiredFields, array $params, $prefixString): void
{
1175 $missingFields = [];
1176 foreach ($requiredFields as $key => $required) {
1177 if (!is_array($required)) {
1178 $importParameter = $params[$key] ??
[];
1179 if (!is_array($importParameter)) {
1180 if (!empty($importParameter)) {
1185 foreach ($importParameter as $locationValues) {
1186 if (!empty($locationValues[$key])) {
1192 $missingFields[$key] = $required;
1195 foreach ($required as $field => $label) {
1196 if (empty($params[$field])) {
1197 $missing[$field] = $label;
1200 if (empty($missing)) {
1203 $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
1206 throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
1210 * Get the field value, transformed by metadata.
1212 * @param string $fieldName
1213 * @param string|int $importedValue
1214 * Value as it came in from the datasource.
1216 * @return string|array|bool|int
1217 * @throws \API_Exception
1219 protected function getTransformedFieldValue(string $fieldName, $importedValue) {
1220 $transformableFields = array_merge($this->metadataHandledFields
, ['country_id']);
1221 // For now only do gender_id etc as we need to work through removing duplicate handling
1222 if (empty($importedValue) ||
!in_array($fieldName, $transformableFields, TRUE)) {
1223 return $importedValue;
1225 $fieldMetadata = $this->getFieldMetadata($fieldName);
1226 if ($fieldName === 'url') {
1227 return CRM_Utils_Rule
::url($importedValue) ?
$importedValue : 'invalid_import_value';
1230 if ($fieldName === 'email') {
1231 return CRM_Utils_Rule
::email($importedValue) ?
$importedValue : 'invalid_import_value';
1234 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_BOOLEAN
) {
1235 $value = CRM_Utils_String
::strtoboolstr($importedValue);
1236 if ($value !== FALSE) {
1237 return (bool) $value;
1239 return 'invalid_import_value';
1241 if ($fieldMetadata['type'] === CRM_Utils_Type
::T_DATE
) {
1242 $value = CRM_Utils_Date
::formatDate($importedValue, $this->getSubmittedValue('dateFormats'));
1243 return ($value) ?
: 'invalid_import_value';
1245 $options = $this->getFieldOptions($fieldName);
1246 if ($options !== FALSE) {
1247 $comparisonValue = is_numeric($importedValue) ?
$importedValue : mb_strtolower($importedValue);
1248 return $options[$comparisonValue] ??
'invalid_import_value';
1250 return $importedValue;
1254 * @param string $fieldName
1256 * @return false|array
1258 * @throws \API_Exception
1260 protected function getFieldOptions(string $fieldName) {
1261 return $this->getFieldMetadata($fieldName, TRUE)['options'];
1265 * Get the metadata for the field.
1267 * @param string $fieldName
1268 * @param bool $loadOptions
1269 * @param bool $limitToContactType
1270 * Only show fields for the type to import (not appropriate when looking up
1271 * related contact fields).
1275 * @throws \API_Exception
1276 * @throws \Civi\API\Exception\NotImplementedException
1278 protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array {
1280 $fieldMap = ['country_id' => 'country'];
1281 $fieldMapName = empty($fieldMap[$fieldName]) ?
$fieldName : $fieldMap[$fieldName];
1283 $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName] ??
($limitToContactType ?
NULL : CRM_Contact_BAO_Contact
::importableFields('All')[$fieldMapName]);
1284 if ($loadOptions && !isset($fieldMetadata['options'])) {
1286 $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
1287 'loadOptions' => ['id', 'name', 'label'],
1288 'where' => [['name', '=', empty($fieldMap[$fieldName]) ?
$fieldMetadata['name'] : $fieldName]],
1289 'select' => ['options'],
1290 ])->first()['options'];
1291 if (is_array($options)) {
1292 // We create an array of the possible variants - notably including
1293 // name AND label as either might be used. We also lower case before checking
1295 foreach ($options as $option) {
1296 $values[$option['id']] = $option['id'];
1297 $values[mb_strtolower($option['name'])] = $option['id'];
1298 $values[mb_strtolower($option['label'])] = $option['id'];
1300 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $values;
1303 $this->importableFieldsMetadata
[$fieldMapName]['options'] = $options;
1305 return $this->importableFieldsMetadata
[$fieldMapName];
1307 return $fieldMetadata;
1311 * @param $customFieldID
1313 * @param array $fieldMetaData
1318 protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?
string {
1319 /* validate the data against the CF type */
1322 $dataType = $fieldMetaData['data_type'];
1323 $htmlType = $fieldMetaData['html_type'];
1324 $isSerialized = CRM_Core_BAO_CustomField
::isSerialized($fieldMetaData);
1325 if ($dataType === 'Date') {
1326 $params = ['date_field' => $value];
1327 if (CRM_Utils_Date
::convertToDefaultDate($params, $dateType, 'date_field')) {
1330 return $fieldMetaData['label'];
1332 elseif ($dataType === 'Boolean') {
1333 if (CRM_Utils_String
::strtoboolstr($value) === FALSE) {
1334 return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle'];
1337 // need not check for label filed import
1338 $selectHtmlTypes = [
1343 if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) ||
$dataType == 'Boolean' ||
$dataType == 'ContactReference') {
1344 $valid = CRM_Core_BAO_CustomValue
::typecheck($dataType, $value);
1346 return $fieldMetaData['label'];
1350 // check for values for custom fields for checkboxes and multiselect
1351 if ($isSerialized && $dataType != 'ContactReference') {
1352 $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen');
1353 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1354 foreach ($mulValues as $v1) {
1357 foreach ($customOption as $v2) {
1358 if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) ||
(strtolower(trim($v2['value'])) == strtolower(trim($v1)))) {
1364 return $fieldMetaData['label'];
1368 elseif ($htmlType == 'Select' ||
($htmlType == 'Radio' && $dataType != 'Boolean')) {
1369 $customOption = CRM_Core_BAO_CustomOption
::getCustomOption($customFieldID, TRUE);
1371 foreach ($customOption as $v2) {
1372 if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) ||
(strtolower(trim($v2['value'])) == strtolower(trim($value)))) {
1377 return $fieldMetaData['label'];
1386 * Get the entity for the given field.
1388 * @param string $fieldName
1390 * @return mixed|null
1391 * @throws \API_Exception
1393 protected function getFieldEntity(string $fieldName) {
1394 if ($fieldName === 'do_not_import') {
1397 $metadata = $this->getFieldMetadata($fieldName);
1398 if (!isset($metadata['entity'])) {
1399 return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ?
'Contact' : $metadata['extends'];
1402 // Our metadata for these is fugly. Handling the fugliness during retrieval.
1403 if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) {
1406 return $metadata['entity'];
1410 * Search the value for the string 'invalid_import_value'.
1412 * If the string is found it indicates the fields was rejected
1413 * during `getTransformedValue` as not having valid data.
1415 * @param string|array|int $value
1416 * @param string $key
1417 * @param string $prefixString
1420 * @throws \API_Exception
1422 protected function getInvalidValues($value, string $key, string $prefixString = ''): array {
1424 if ($value === 'invalid_import_value') {
1425 $errors[] = $prefixString . $this->getFieldMetadata($key)['title'];
1427 elseif (is_array($value)) {
1428 foreach ($value as $innerKey => $innerValue) {
1429 $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString);
1430 if (!empty($result)) {
1431 $errors = array_merge($result, $errors);
1435 return array_filter($errors);
1439 * Get the available countries.
1441 * If the site is not configured with a restriction then all countries are valid
1442 * but otherwise only a select array are.
1444 * @return array|false
1445 * FALSE indicates no restrictions.
1447 protected function getAvailableCountries() {
1448 if ($this->availableCountries
=== NULL) {
1449 $availableCountries = Civi
::settings()->get('countryLimit');
1450 $this->availableCountries
= !empty($availableCountries) ?
array_fill_keys($availableCountries, TRUE) : FALSE;
1452 return $this->availableCountries
;