From: Eileen McNaughton Date: Thu, 1 Sep 2022 02:38:13 +0000 (+1200) Subject: Switch to api metadata for contact on non-contact imports X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=bc4aa49d6c529fe3ccbc24c339660518d873f73f;p=civicrm-core.git Switch to api metadata for contact on non-contact imports This is kinda 'safe' in that these imports only support non-unique contact fields --- diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 04c7c91f2f..11f59ffb5a 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -1059,7 +1059,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { * @throws \CiviCRM_API3_Exception */ protected function getPossibleContactMatch(array $params, ?int $extIDMatch, $dedupeRuleID): ?int { - $possibleMatches = $this->getPossibleMatchesByDedupeRule($params, $dedupeRuleID); + $possibleMatches = $this->getPossibleMatchesByDedupeRule($params, $dedupeRuleID, FALSE); if (!$extIDMatch) { if (count($possibleMatches) === 1) { return array_key_last($possibleMatches); diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index a67a94574d..e7853408e8 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -200,14 +200,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { else { $fieldSpec = $this->getFieldMetadata($mappedField['name']); $entity = $fieldSpec['entity'] ?? 'Contribution'; - if ($fieldSpec['hasLocationType'] ?? NULL) { - $fieldEntity = str_replace('civicrm_', '', $fieldSpec['table_name']); - $fieldName = $fieldEntity . '_primary.' . $this->getFieldMetadata($mappedField['name'])['name']; - $params[$entity][$fieldName] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); - } - else { - $params[$entity][$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); - } + $params[$entity][$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); } } return $params; diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index d780929d60..40aaee0530 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -9,12 +9,15 @@ +--------------------------------------------------------------------+ */ +use Civi\Api4\Address; use Civi\Api4\Campaign; use Civi\Api4\Contact; use Civi\Api4\CustomField; use Civi\Api4\DedupeRule; use Civi\Api4\DedupeRuleGroup; +use Civi\Api4\Email; use Civi\Api4\Event; +use Civi\Api4\Phone; use Civi\Api4\UserJob; use Civi\UserJob\UserJobInterface; @@ -316,7 +319,7 @@ abstract class CRM_Import_Parser implements UserJobInterface { * @return array[] */ protected function getContactFields(string $contactType): array { - $contactFields = CRM_Contact_BAO_Contact::importableFields($contactType, NULL); + $contactFields = $this->getAllContactFields(''); $dedupeFields = $this->getDedupeFields($contactType); $contactFieldsForContactLookup = []; @@ -2233,34 +2236,40 @@ abstract class CRM_Import_Parser implements UserJobInterface { * * @param array $params * @param int|null $dedupeRuleID + * @param bool $isApiMetadata + * Is the import using api4 style metadata (in which case no conversion needed) - eventually + * only contact import will use a different style (as it supports multiple locations) and the + * handling will be in that class. * * @return array * * @throws \CRM_Core_Exception */ - protected function getPossibleMatchesByDedupeRule(array $params, $dedupeRuleID = NULL): array { - foreach (['email', 'address', 'phone', 'im'] as $locationEntity) { - if (array_key_exists($locationEntity, $params)) { - // Prefer primary - if (array_key_exists('Primary', $params[$locationEntity])) { - $locationParams = $params[$locationEntity]['Primary']; - } - else { - // Chose the first one - at least they can manipulate the order. - $locationParams = reset($params[$locationEntity]); - } - foreach ($locationParams as $key => $locationParam) { - // Even though we might not be using 'primary' we 'pretend' here - // since the apiv4 code expects that... - $params[$locationEntity . '_primary' . '.' . $key] = $locationParam; + protected function getPossibleMatchesByDedupeRule(array $params, $dedupeRuleID = NULL, $isApiMetadata = TRUE): array { + if ($isApiMetadata === FALSE) { + foreach (['email', 'address', 'phone', 'im'] as $locationEntity) { + if (array_key_exists($locationEntity, $params)) { + // Prefer primary + if (array_key_exists('Primary', $params[$locationEntity])) { + $locationParams = $params[$locationEntity]['Primary']; + } + else { + // Chose the first one - at least they can manipulate the order. + $locationParams = reset($params[$locationEntity]); + } + foreach ($locationParams as $key => $locationParam) { + // Even though we might not be using 'primary' we 'pretend' here + // since the apiv4 code expects that... + $params[$locationEntity . '_primary' . '.' . $key] = $locationParam; + } + unset($params[$locationEntity]); } - unset($params[$locationEntity]); } - } - foreach ($params as $key => $value) { - if (strpos($key, 'custom_') === 0) { - $params[$this->getApi4Name($key)] = $value; - unset($params[$key]); + foreach ($params as $key => $value) { + if (strpos($key, 'custom_') === 0) { + $params[$this->getApi4Name($key)] = $value; + unset($params[$key]); + } } } $dedupeRule = $dedupeRuleID ? $this->getDedupeRuleName($dedupeRuleID) : $this->getDefaultRuleForContactType($params['contact_type']); @@ -2346,4 +2355,108 @@ abstract class CRM_Import_Parser implements UserJobInterface { return $this->getDedupeRule($contactType)['fields']; } + /** + * Get all contact import fields metadata. + * + * @param string $prefix + * + * @return array + * + * @noinspection PhpUnhandledExceptionInspection + */ + protected function getAllContactFields(string $prefix = 'Contact.'): array { + $allContactFields = (array) Contact::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + ->addWhere('fk_entity', 'IS EMPTY') + ->addOrderBy('title') + ->execute()->indexBy('name'); + + $contactTypeFields['Individual'] = (array) Contact::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + ->addWhere('fk_entity', 'IS EMPTY') + ->setSelect(['name']) + ->addValue('contact_type', 'Individual') + ->addOrderBy('title') + ->execute()->indexBy('name'); + + $contactTypeFields['Organization'] = (array) Contact::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + ->addWhere('fk_entity', 'IS EMPTY') + ->setSelect(['name']) + ->addValue('contact_type', 'Organization') + ->addOrderBy('title') + ->execute()->indexBy('name'); + + $contactTypeFields['Household'] = (array) Contact::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + ->addWhere('fk_entity', 'IS EMPTY') + ->setSelect(['name']) + ->addOrderBy('title') + ->execute()->indexBy('name'); + + $prefixedFields = []; + foreach ($allContactFields as $fieldName => $field) { + $field['contact_type'] = []; + foreach ($contactTypeFields as $contactTypeName => $fields) { + if (array_key_exists($fieldName, $fields)) { + $field['contact_type'][$contactTypeName] = $contactTypeName; + } + } + $fieldName = $prefix . $fieldName; + if (!empty($field['custom_field_id'])) { + $this->customFieldNameMap['custom_' . $field['custom_field_id']] = $fieldName; + } + $prefixedFields[$fieldName] = $field; + } + + $addressFields = (array) Address::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + ->addOrderBy('title') + // Exclude these fields to keep it simpler for now - we just map to primary + ->addWhere('name', 'NOT IN', ['id', 'location_type_id', 'master_id']) + ->execute()->indexBy('name'); + foreach ($addressFields as $fieldName => $field) { + // Set entity to contact as primary fields used in Contact actions + $field['entity'] = 'Contact'; + $field['name'] = 'address_primary.' . $fieldName; + $field['contact_type'] = ['Individual', 'Organization', 'Household']; + $prefixedFields[$prefix . 'address_primary.' . $fieldName] = $field; + } + + $phoneFields = (array) Phone::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + // Exclude these fields to keep it simpler for now - we just map to primary + ->addWhere('name', 'NOT IN', ['id', 'location_type_id', 'phone_type_id']) + ->addOrderBy('title') + ->execute()->indexBy('name'); + foreach ($phoneFields as $fieldName => $field) { + $field['entity'] = 'Contact'; + $field['name'] = 'phone_primary.' . $fieldName; + $field['contact_type'] = ['Individual', 'Organization', 'Household']; + $prefixedFields[$prefix . 'phone_primary.' . $fieldName] = $field; + } + + $emailFields = (array) Email::getFields() + ->addWhere('readonly', '=', FALSE) + ->addWhere('type', 'IN', ['Field', 'Custom']) + // Exclude these fields to keep it simpler for now - we just map to primary + ->addWhere('name', 'NOT IN', ['id', 'location_type_id']) + ->addOrderBy('title') + ->execute()->indexBy('name'); + + foreach ($emailFields as $fieldName => $field) { + $field['entity'] = 'Contact'; + $field['name'] = 'email_primary.' . $fieldName; + $field['contact_type'] = ['Individual', 'Organization', 'Household']; + $prefixedFields[$prefix . 'email_primary.' . $fieldName] = $field; + } + return $prefixedFields; + } + } diff --git a/CRM/Upgrade/Incremental/php/FiveFiftyFour.php b/CRM/Upgrade/Incremental/php/FiveFiftyFour.php index 166155e109..c70267b05c 100644 --- a/CRM/Upgrade/Incremental/php/FiveFiftyFour.php +++ b/CRM/Upgrade/Incremental/php/FiveFiftyFour.php @@ -44,6 +44,7 @@ class CRM_Upgrade_Incremental_php_FiveFiftyFour extends CRM_Upgrade_Incremental_ $this->addTask('Increase field length of civicrm_dedupe_rule_group.name', 'alterDedupeRuleGroupName'); $this->addTask('Add index civicrm_dedupe_rule_group.UI_name', 'addIndex', 'civicrm_dedupe_rule_group', 'name', 'UI'); $this->addTask('Install Elavon Payment Processor Extension as needed', 'installElavonPaymentProcessorExtension'); + $this->addTask('Convert field names for contribution import saved mappings', 'updateContributionMappings'); } public static function addCreatedIDColumnToParticipant($ctx): bool { @@ -146,4 +147,63 @@ class CRM_Upgrade_Incremental_php_FiveFiftyFour extends CRM_Upgrade_Incremental_ return TRUE; } + /** + * Update saved mappings for contribution imports to use apiv4 style field names. + * + * In time we will do this to the other imports. + * + * @return true + */ + public static function updateContributionMappings(): bool { + $mappingTypeID = (int) CRM_Core_DAO::singleValueQuery(" + SELECT option_value.value + FROM civicrm_option_value option_value + INNER JOIN civicrm_option_group option_group + ON option_group.id = option_value.option_group_id + AND option_group.name = 'mapping_type' + WHERE option_value.name = 'Import Contribution'"); + + $mappingFields = CRM_Core_DAO::executeQuery(' + SELECT field.id, field.name FROM civicrm_mapping_field field + INNER JOIN civicrm_mapping mapping + ON field.mapping_id = mapping.id + AND mapping_type_id = ' . $mappingTypeID + ); + // Only dedupe fields could be stored. Phone number, email, address fields & custom fields + // is a realistic set. The impact of missing something is pretty minor as saved field mappings + // are easy to update during import & people normally do a visual check - so hard coding a list + // feels more future-proof than doing it by code. + $fieldsToConvert = [ + 'email' => 'email_primary.email', + 'phone' => 'phone_primary.phone', + 'street_address' => 'address_primary.street_address', + 'supplemental_address_1' => 'address_primary.supplemental_address_1', + 'supplemental_address_2' => 'address_primary.supplemental_address_2', + 'supplemental_address_3' => 'address_primary.supplemental_address_3', + 'city' => 'address_primary.city', + 'county_id' => 'address_primary.county_id', + 'state_province_id' => 'address_primary.state_province_id', + 'country_id' => 'address_primary.country_id', + ]; + $customFields = CRM_Core_DAO::executeQuery(' + SELECT custom_field.id, custom_field.name, custom_group.name as custom_group_name + FROM civicrm_custom_field custom_field INNER JOIN civicrm_custom_group custom_group + ON custom_field.custom_group_id = custom_group.id + WHERE extends IN ("Contact", "Individual", "Organization", "Household") + '); + while ($customFields->fetch()) { + $fieldsToConvert['custom_' . $customFields->id] = $customFields->custom_group_name . '.' . $customFields->name; + } + while ($mappingFields->fetch()) { + // Convert the field. + if (isset($fieldsToConvert[$mappingFields->name])) { + CRM_Core_DAO::executeQuery(' UPDATE civicrm_mapping_field SET name = %1 WHERE id = %2', [ + 1 => [$fieldsToConvert[$mappingFields->name], 'String'], + 2 => [$mappingFields->id, 'Integer'], + ]); + } + } + return TRUE; + } + } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index babfd41f96..85dc689dfc 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -195,7 +195,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $contactID = $this->individualCreate(['email' => 'mum@example.com']); $pledgeID = $this->pledgeCreate(['contact_id' => $contactID]); $this->importCSV('pledge.csv', [ - ['name' => 'email'], + ['name' => 'email_primary.email'], ['name' => 'total_amount'], ['name' => 'pledge_id'], ['name' => 'receive_date'], @@ -242,7 +242,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $parser = new CRM_Contribute_Import_Parser_Contribution(); $parser->setUserJobID($this->getUserJobID()); $fields = $parser->getFieldsMetadata(); - $this->assertArrayHasKey('phone', $fields); + $this->assertArrayHasKey('phone_primary.phone', $fields); $this->callApiSuccess('RuleGroup', 'create', [ 'id' => $unsupervisedRuleGroup['id'], 'used' => 'Unsupervised', @@ -396,7 +396,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => ''], ['name' => 'receive_date'], ['name' => 'financial_type_id'], - ['name' => 'email'], + ['name' => 'email_primary.email'], ['name' => ''], ['name' => ''], ['name' => 'trxn_id'], @@ -662,7 +662,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => 'total_amount'], ['name' => 'receive_date'], ['name' => 'financial_type_id'], - ['name' => 'email'], + ['name' => 'email_primary.email'], ['name' => 'contribution_source'], ['name' => 'note'], ['name' => 'trxn_id'],