From 4601be8b0983750c2e7655b6e1c33892cd50fab3 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Tue, 21 Mar 2023 17:11:54 +1300 Subject: [PATCH] Update Contribution Import to use apiv4 field names, prior to adding hooks --- CRM/Contribute/Import/Parser/Contribution.php | 26 ++++++++++-- CRM/Import/Parser.php | 26 +++++++++--- CRM/Upgrade/Incremental/php/FiveSixtyOne.php | 38 ++++++++++++++++- .../Import/Parser/ContributionTest.php | 41 ++++++++++++------- 4 files changed, 105 insertions(+), 26 deletions(-) diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index ebd72604d3..b0dd2a59f0 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -247,14 +247,27 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $note = CRM_Core_DAO_Note::import(); $tmpFields = CRM_Contribute_DAO_Contribution::import(); + // Unravel the unique fields - once more metadata work is done on apiv4 we + // will use that instead to get them. + foreach (['contribution_cancel_date', 'contribution_check_number', 'contribution_campaign_id'] as $uniqueField) { + $realField = substr($uniqueField, 13); + $tmpFields[$realField] = $tmpFields[$uniqueField]; + unset($tmpFields[$uniqueField]); + } $tmpContactField = $this->getContactFields($this->getContactType()); + // I haven't un-done this unique field yet cos it's more complex. $tmpFields['contribution_contact_id']['title'] = $tmpFields['contribution_contact_id']['html']['label'] = $tmpFields['contribution_contact_id']['title'] . ' ' . ts('(match to contact)'); $tmpFields['contribution_contact_id']['contact_type'] = ['Individual' => 'Individual', 'Household' => 'Household', 'Organization' => 'Organization']; $tmpFields['contribution_contact_id']['match_rule'] = '*'; $fields = array_merge($fields, $tmpContactField); $fields = array_merge($fields, $tmpFields); $fields = array_merge($fields, $note); - $fields = array_merge($fields, CRM_Core_BAO_CustomField::getFieldsForImport('Contribution')); + $apiv4 = Contribution::getFields(TRUE)->addWhere('custom_field_id', '>', 0)->execute(); + $customFields = []; + foreach ($apiv4 as $apiv4Field) { + $customFields[$apiv4Field['name']] = $apiv4Field; + } + $fields = array_merge($fields, $customFields); $fields['soft_credit.contact.id'] = [ 'title' => ts('Soft Credit Contact ID'), @@ -384,7 +397,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $rowNumber = (int) ($values[array_key_last($values)]); try { $params = $this->getMappedRow($values); - $contributionParams = array_merge(['version' => 3, 'skipRecentView' => TRUE, 'skipCleanMoney' => TRUE], $params['Contribution']); + $contributionParams = $params['Contribution']; //CRM-10994 if (isset($contributionParams['total_amount']) && $contributionParams['total_amount'] == 0) { $contributionParams['total_amount'] = '0.00'; @@ -424,7 +437,12 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $this->deleteExistingSoftCredit($contributionParams['id']); } - $contributionID = civicrm_api3('contribution', 'create', $contributionParams)['id']; + if ($contributionParams['id']) { + $contributionID = Contribution::update()->setValues($contributionParams)->execute()->first()['id']; + } + else { + $contributionID = Contribution::create()->setValues($contributionParams)->execute()->first()['id']; + } if (!empty($softCreditParams)) { if (empty($contributionParams['total_amount']) || empty($contributionParams['currency'])) { @@ -732,7 +750,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * @return string[] */ protected function getOddlyMappedMetadataFields(): array { - $uniqueNames = ['contribution_id', 'contribution_contact_id', 'contribution_cancel_date', 'contribution_source', 'contribution_check_number']; + $uniqueNames = ['contribution_id', 'contribution_contact_id', 'contribution_source']; $fields = []; foreach ($uniqueNames as $name) { $fields[$this->importableFieldsMetadata[$name]['name']] = $name; diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 7ba284499b..88ff6690ea 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -1594,20 +1594,34 @@ abstract class CRM_Import_Parser implements UserJobInterface { return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value'; } - if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) { + // DataType is defined on apiv4 metadata - ie what we are moving to. + $typeMap = [ + CRM_Utils_Type::T_FLOAT => 'Float', + CRM_Utils_Type::T_MONEY => 'Money', + CRM_Utils_Type::T_BOOLEAN => 'Boolean', + CRM_Utils_Type::T_DATE => 'Date', + (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) => 'Timestamp', + CRM_Utils_Type::T_TIMESTAMP => 'Timestamp', + CRM_Utils_Type::T_INT => 'Integer', + CRM_Utils_Type::T_TEXT => 'String', + CRM_Utils_Type::T_STRING => 'String', + ]; + $dataType = $fieldMetadata['data_type'] ?? $typeMap[$fieldMetadata['type']]; + + if ($dataType === 'Float') { return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value'; } - if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) { + if ($dataType === 'Money') { return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value'; } - if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) { + if ($dataType === 'Boolean') { $value = CRM_Utils_String::strtoboolstr($importedValue); if ($value !== FALSE) { return (bool) $value; } return 'invalid_import_value'; } - if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) { + if (in_array($dataType, ['Date', 'Timestamp'], TRUE)) { $value = CRM_Utils_Date::formatDate($importedValue, (int) $this->getSubmittedValue('dateFormats')); return $value ?: 'invalid_import_value'; } @@ -1640,7 +1654,7 @@ abstract class CRM_Import_Parser implements UserJobInterface { return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value'; } } - if ($fieldMetadata['type'] === CRM_Utils_Type::T_INT) { + if ($dataType === 'Integer') { // We have resolved the options now so any remaining ones should be integers. return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value'; } @@ -1685,7 +1699,7 @@ abstract class CRM_Import_Parser implements UserJobInterface { } $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName]; - if ($loadOptions && !isset($fieldMetadata['options'])) { + if ($loadOptions && (!isset($fieldMetadata['options']) || $fieldMetadata['options'] === TRUE)) { if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') { // Probably already loaded and also supports abbreviations - eg. NSW. // Supporting for core AND custom state fields is more consistent. diff --git a/CRM/Upgrade/Incremental/php/FiveSixtyOne.php b/CRM/Upgrade/Incremental/php/FiveSixtyOne.php index 31b99a7965..caa0ade7dd 100644 --- a/CRM/Upgrade/Incremental/php/FiveSixtyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveSixtyOne.php @@ -9,6 +9,9 @@ +--------------------------------------------------------------------+ */ +use Civi\Api4\Contribution; +use Civi\Api4\MappingField; + /** * Upgrade logic for the 5.61.x series. * @@ -50,6 +53,7 @@ class CRM_Upgrade_Incremental_php_FiveSixtyOne extends CRM_Upgrade_Incremental_B $this->addTask(ts('Drop index %1', [1 => 'civicrm_cache.UI_group_path_date']), 'dropIndex', 'civicrm_cache', 'UI_group_path_date'); $this->addTask(ts('Create index %1', [1 => 'civicrm_cache.UI_group_name_path']), 'addIndex', 'civicrm_cache', [['group_name', 'path']], 'UI'); $this->addTask(ts('Create index %1', [1 => 'civicrm_cache.index_expired_date']), 'addIndex', 'civicrm_cache', [['expired_date']], 'index'); + $this->addTask(ts('Update Saved Mapping for contribution import', [1 => $rev]), 'convertMappingFieldsToApi4StyleNames', $rev); } /** @@ -67,7 +71,7 @@ class CRM_Upgrade_Incremental_php_FiveSixtyOne extends CRM_Upgrade_Incremental_B public static function dedupeCache($ctx): bool { $duplicates = CRM_Core_DAO::executeQuery(' SELECT c.id FROM civicrm_cache c - LEFT JOIN (SELECT group_name, path, max(created_date) newest FROM civicrm_cache GROUP BY group_name, path) recent + LEFT JOIN (SELECT group_name, path, MAX(created_date) newest FROM civicrm_cache GROUP BY group_name, path) recent ON (c.group_name=recent.group_name AND c.path=recent.path AND c.created_date=recent.newest) WHERE recent.newest IS NULL') ->fetchMap('id', 'id'); @@ -77,7 +81,39 @@ class CRM_Upgrade_Incremental_php_FiveSixtyOne extends CRM_Upgrade_Incremental_B ->param('IDS', $duplicates) ->execute(); } + return TRUE; + } + + /** + * @return bool + * @throws \CRM_Core_Exception + * @noinspection PhpUnused + */ + public static function convertMappingFieldsToApi4StyleNames(): bool { + $mappings = MappingField::get(FALSE) + ->setSelect(['id', 'name']) + ->addWhere('mapping_id.mapping_type_id:name', '=', 'Import Contribution') + ->execute(); + + $fieldMap = [ + 'contribution_cancel_date' => 'cancel_date', + 'contribution_check_number' => 'check_number', + 'contribution_campaign_id' => 'campaign_id', + ]; + $apiv4 = Contribution::getFields(FALSE)->addWhere('custom_field_id', '>', 0)->execute(); + foreach ($apiv4 as $apiv4Field) { + $fieldMap['custom_' . $apiv4Field['id']] = $apiv4Field['name']; + } + // Update the mapped fields. + foreach ($mappings as $mapping) { + if (!empty($fieldMap[$mapping['name']])) { + MappingField::update(FALSE) + ->addWhere('id', '=', $mapping['id']) + ->addValue('name', $fieldMap[$mapping['name']]) + ->execute(); + } + } return TRUE; } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index caaa624032..59fe47fb44 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -270,7 +270,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { // Note that default_value is supported via the parser and the angular form // but there is no way to enter it on the quick form. ['name' => 'financial_type_id', 'default_value' => 'Donation'], - ['name' => 'contribution_source'], + ['name' => 'source'], ['name' => 'receive_date'], ['name' => 'external_identifier'], ['name' => 'soft_credit.contact.email_primary.email', 'entity_data' => ['soft_credit' => ['soft_credit_type_id' => 5]]], @@ -336,7 +336,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => 'receive_date'], ['name' => 'financial_type_id'], ['name' => 'external_identifier'], - ['name' => $this->getCustomFieldName('date')], + ['name' => $this->getCustomFieldName('date', 4)], ]; $this->importCSV('contributions_date_validate.csv', $mapping, ['dateFormats' => 32]); $contribution = $this->callAPISuccessGetSingle('Contribution', []); @@ -355,7 +355,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $contribution = $this->callAPISuccess('Contribution', 'getsingle', ['contact_id' => $contactID]); $this->createCustomGroupWithFieldOfType([], 'radio'); $values['contribution_id'] = $contribution['id']; - $values[$this->getCustomFieldName('radio')] = 'Red Testing'; + $values[$this->getCustomFieldName('radio', 4)] = 'Red Testing'; unset(Civi::$statics['CRM_Core_BAO_OptionGroup']); $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE); $contribution = $this->callAPISuccess('Contribution', 'get', ['contact_id' => $contactID, $this->getCustomFieldName('radio') => 'Red Testing']); @@ -391,11 +391,11 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { */ public function testPhoneMatchOnContact(): void { // Update existing unsupervised rule, change to general. - $unsupervisedRuleGroup = $this->callApiSuccess('RuleGroup', 'getsingle', [ + $unsupervisedRuleGroup = $this->callAPISuccess('RuleGroup', 'getsingle', [ 'used' => 'Unsupervised', 'contact_type' => 'Individual', ]); - $this->callApiSuccess('RuleGroup', 'create', [ + $this->callAPISuccess('RuleGroup', 'create', [ 'id' => $unsupervisedRuleGroup['id'], 'used' => 'General', ]); @@ -419,7 +419,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $parser->setUserJobID($this->getUserJobID()); $fields = $parser->getFieldsMetadata(); $this->assertArrayHasKey('phone_primary.phone', $fields); - $this->callApiSuccess('RuleGroup', 'create', [ + $this->callAPISuccess('RuleGroup', 'create', [ 'id' => $unsupervisedRuleGroup['id'], 'used' => 'Unsupervised', ]); @@ -434,11 +434,13 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { */ public function testCustomSerializedCheckBox(): void { $this->createCustomGroupWithFieldOfType([], 'checkbox'); - $customField = $this->getCustomFieldName('checkbox'); + $customField = $this->getCustomFieldName('checkbox', 4); $contactID = $this->individualCreate(); $values = ['contribution_contact_id' => $contactID, 'total_amount' => 10, 'financial_type_id' => 'Donation', $customField => 'L,V']; $this->runImport($values, CRM_Import_Parser::DUPLICATE_SKIP); - $initialContribution = $this->callAPISuccessGetSingle('Contribution', ['contact_id' => $contactID]); + $initialContribution = Contribution::get()->addWhere('contact_id', '=', $contactID) + ->addSelect($customField) + ->execute()->first(); $this->assertContains('L', $initialContribution[$customField], 'Contribution Duplicate Skip Import contains L'); $this->assertContains('V', $initialContribution[$customField], 'Contribution Duplicate Skip Import contains V'); @@ -447,7 +449,10 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $values[$customField] = 'V'; $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE); - $updatedContribution = $this->callAPISuccessGetSingle('Contribution', ['id' => $initialContribution['id']]); + $updatedContribution = Contribution::get()->addWhere('id', '=', $initialContribution['id']) + ->addSelect($customField) + ->execute()->first(); + $this->assertNotContains('L', $updatedContribution[$customField], 'Contribution Duplicate Update Import does not contain L'); $this->assertContains('V', $updatedContribution[$customField], 'Contribution Duplicate Update Import contains V'); @@ -596,7 +601,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => ''], ['name' => ''], ['name' => 'trxn_id'], - ['name' => 'contribution_campaign_id'], + ['name' => 'campaign_id'], ['name' => 'contribution_contact_id'], ]; // First we try to create without total_amount mapped. @@ -629,9 +634,9 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => ''], ['name' => ''], ['name' => ''], - ['name' => 'contribution_source'], + ['name' => 'source'], ['name' => 'trxn_id'], - ['name' => 'contribution_campaign_id'], + ['name' => 'campaign_id'], ]; $this->importCSV('contributions.csv', $fieldMappings, ['onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE]); @@ -719,7 +724,13 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { 'mapper' => $mapper, ])); $parser->init(); - $parser->import($values); + try { + $parser->validateValues($values); + $parser->import($values); + } + catch (CRM_Core_Exception $e) { + // Validation failed. + } } /** @@ -776,7 +787,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => 'total_amount'], ['name' => 'receive_date'], ['name' => 'financial_type_id'], - ['name' => 'contribution_source'], + ['name' => 'source'], ['name' => ''], ]; $this->importCSV('contributions_update.csv', $mapping, ['onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE]); @@ -863,7 +874,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ['name' => 'receive_date'], ['name' => 'financial_type_id'], ['name' => 'email_primary.email'], - ['name' => 'contribution_source'], + ['name' => 'source'], ['name' => 'note'], ['name' => 'trxn_id'], ], $submittedValues); -- 2.25.1