From 73edfc10782259e5fc0a1472c2506e3ce0092c4a Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Fri, 20 May 2022 17:47:47 +1200 Subject: [PATCH] [Import] Fix Contribution Import mapping fields to use labels --- CRM/Contact/Import/Form/MapField.php | 53 ++------- CRM/Contact/Import/Form/Preview.php | 9 ++ CRM/Contact/Import/Parser/Contact.php | 39 ------- CRM/Contribute/Import/Form/MapField.php | 58 ++++------ CRM/Contribute/Import/Parser/Contribution.php | 107 +++++++++++++----- CRM/Import/Form/MapField.php | 40 +++++++ CRM/Import/Forms.php | 13 ++- CRM/Import/Parser.php | 39 +++++++ CRM/Upgrade/Incremental/php/FiveFiftyOne.php | 40 +++++++ 9 files changed, 243 insertions(+), 155 deletions(-) diff --git a/CRM/Contact/Import/Form/MapField.php b/CRM/Contact/Import/Form/MapField.php index 24ce0d22bf..688012fb8c 100644 --- a/CRM/Contact/Import/Form/MapField.php +++ b/CRM/Contact/Import/Form/MapField.php @@ -15,8 +15,6 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ -use Civi\Api4\MappingField; - /** * This class gets the name of the file to upload. */ @@ -458,7 +456,7 @@ class CRM_Contact_Import_Form_MapField extends CRM_Import_Form_MapField { //Updating Mapping Records if (!empty($params['updateMapping'])) { - for ($i = 0; $i < $this->_columnCount; $i++) { + foreach (array_keys($this->getColumnHeaders()) as $i) { $this->saveMappingField($params['mappingId'], $i, TRUE); } } @@ -489,46 +487,6 @@ class CRM_Contact_Import_Form_MapField extends CRM_Import_Form_MapField { return $parser; } - /** - * Save the mapping field. - * - * @param int $mappingID - * @param int $columnNumber - * @param bool $isUpdate - * - * @throws \API_Exception - * @throws \CRM_Core_Exception - */ - protected function saveMappingField(int $mappingID, int $columnNumber, bool $isUpdate = FALSE): void { - $fieldMapping = (array) $this->getSubmittedValue('mapper')[$columnNumber]; - $mappedField = $this->getMappedField($fieldMapping, $mappingID, $columnNumber); - if ($isUpdate) { - MappingField::update(FALSE) - ->setValues($mappedField) - ->addWhere('column_number', '=', $columnNumber) - ->addWhere('mapping_id', '=', $mappingID) - ->execute(); - } - else { - MappingField::create(FALSE) - ->setValues($mappedField)->execute(); - } - } - - /** - * Get the field mapped to the savable format. - * - * @param array $fieldMapping - * @param int $mappingID - * @param int $columnNumber - * - * @return array - * @throws \CRM_Core_Exception - */ - protected function getMappedField(array $fieldMapping, int $mappingID, int $columnNumber): array { - return (new CRM_Contact_Import_Parser_Contact())->setContactType($this->getContactType())->getMappingFieldFromMapperInput($fieldMapping, $mappingID, $columnNumber); - } - /** * Did the user specify duplicates matching should not be attempted. * @@ -595,4 +553,13 @@ class CRM_Contact_Import_Form_MapField extends CRM_Import_Form_MapField { return (bool) (Civi::$statics[__CLASS__]['location_fields'][$name] ?? FALSE); } + /** + * @return \CRM_Contact_Import_Parser_Contact + */ + protected function getParser(): CRM_Contact_Import_Parser_Contact { + $parser = new CRM_Contact_Import_Parser_Contact(); + $parser->setUserJobID($this->getUserJobID()); + return $parser; + } + } diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index f51f2b5405..9f03057aee 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -261,4 +261,13 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { return $mapper; } + /** + * @return \CRM_Contact_Import_Parser_Contact + */ + protected function getParser(): CRM_Contact_Import_Parser_Contact { + $parser = new CRM_Contact_Import_Parser_Contact(); + $parser->setUserJobID($this->getUserJobID()); + return $parser; + } + } diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 5813f6a8b4..6b120617fd 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -181,45 +181,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { ], TRUE); } - /** - * Gets the fields available for importing in a key-name, title format. - * - * @return array - * eg. ['first_name' => 'First Name'.....] - * - * @throws \API_Exception - * - * @todo - we are constructing the metadata before we - * have set the contact type so we re-do it here. - * - * Once we have cleaned up the way the mapper is handled - * we can ditch all the existing _construct parameters in favour - * of just the userJobID - there are current open PRs towards this end. - */ - public function getAvailableFields(): array { - $this->setFieldMetadata(); - $return = []; - foreach ($this->getImportableFieldsMetadata() as $name => $field) { - if ($name === 'id' && $this->isSkipDuplicates()) { - // Duplicates are being skipped so id matching is not availble. - continue; - } - $return[$name] = $field['title']; - } - return $return; - } - - /** - * Did the user specify duplicates should be skipped and not imported. - * - * @return bool - * - * @throws \API_Exception - */ - private function isSkipDuplicates(): bool { - return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP; - } - /** * Did the user specify duplicates checking should be skipped, resulting in possible duplicate contacts. * diff --git a/CRM/Contribute/Import/Form/MapField.php b/CRM/Contribute/Import/Form/MapField.php index 10c4144cb1..34567d0874 100644 --- a/CRM/Contribute/Import/Form/MapField.php +++ b/CRM/Contribute/Import/Form/MapField.php @@ -72,7 +72,7 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { * Set variables up before form is built. */ public function preProcess() { - $this->_mapperFields = $this->get('fields'); + $this->_mapperFields = $this->getAvailableFields(); asort($this->_mapperFields); $this->_columnCount = $this->get('columnCount'); @@ -198,11 +198,8 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $mappingName = $mappingName[1]; $mappingContactType = $mappingContactType[1]; if (isset($mappingName[$i])) { - if ($mappingName[$i] != ts('- do not import -')) { - - $mappingHeader = array_keys($this->_mapperFields, $mappingName[$i]); - // reusing contact_type field array for soft credit - $softField = $mappingContactType[$i] ?? 0; + if ($mappingName[$i] != ts('do_not_import')) { + $softField = $mappingContactType[$i] ?? ''; if (!$softField) { $js .= "{$formName}['mapper[$i][1]'].style.display = 'none';\n"; @@ -211,10 +208,10 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $js .= "{$formName}['mapper[$i][2]'].style.display = 'none';\n"; $js .= "{$formName}['mapper[$i][3]'].style.display = 'none';\n"; $defaults["mapper[$i]"] = [ - CRM_Utils_Array::value(0, $mappingHeader), - ($softField) ? $softField : "", - "", - "", + $mappingName[$i], + $softField, + // Since the soft credit type id is not stored we can't load it here. + '', ]; $jsSet = TRUE; } @@ -461,27 +458,8 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { //Updating Mapping Records if (!empty($params['updateMapping'])) { - $mappingFields = new CRM_Core_DAO_MappingField(); - $mappingFields->mapping_id = $params['mappingId']; - $mappingFields->find(); - - $mappingFieldsId = []; - while ($mappingFields->fetch()) { - if ($mappingFields->id) { - $mappingFieldsId[$mappingFields->column_number] = $mappingFields->id; - } - } - for ($i = 0; $i < $this->_columnCount; $i++) { - $updateMappingFields = new CRM_Core_DAO_MappingField(); - $updateMappingFields->id = $mappingFieldsId[$i]; - $updateMappingFields->mapping_id = $params['mappingId']; - $updateMappingFields->column_number = $i; - $updateMappingFields->name = $mapper[$i]; - - //reuse contact_type field in db to store fields associated with soft credit - $updateMappingFields->contact_type = $mapperSoftCredit[$i] ?? NULL; - $updateMappingFields->save(); + $this->saveMappingField($params['mappingId'], $i, TRUE); } } @@ -495,16 +473,9 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $saveMapping = CRM_Core_BAO_Mapping::add($mappingParams); for ($i = 0; $i < $this->_columnCount; $i++) { - $saveMappingFields = new CRM_Core_DAO_MappingField(); - $saveMappingFields->mapping_id = $saveMapping->id; - $saveMappingFields->column_number = $i; - $saveMappingFields->name = $mapper[$i]; - - //reuse contact_type field in db to store fields associated with soft credit - $saveMappingFields->contact_type = $mapperSoftCredit[$i] ?? NULL; - $saveMappingFields->save(); + $this->saveMappingField($saveMapping->id, $i, FALSE); } - $this->set('savedMapping', $saveMappingFields->mapping_id); + $this->set('savedMapping', $saveMapping->id); } $parser = new CRM_Contribute_Import_Parser_Contribution($mapperKeysMain, $mapperSoftCredit, $mapperPhoneType); @@ -521,4 +492,13 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $parser->set($this); } + /** + * @return \CRM_Contribute_Import_Parser_Contribution + */ + protected function getParser(): CRM_Contribute_Import_Parser_Contribution { + $parser = new CRM_Contribute_Import_Parser_Contribution(); + $parser->setUserJobID($this->getUserJobID()); + return $parser; + } + } diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index bed232eb46..d97504e9f4 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -657,37 +657,8 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * The initializer code, called before the processing */ public function init() { - $fields = CRM_Contribute_BAO_Contribution::importableFields($this->_contactType, FALSE); - - $fields = array_merge($fields, - [ - 'soft_credit' => [ - 'title' => ts('Soft Credit'), - 'softCredit' => TRUE, - 'headerPattern' => '/Soft Credit/i', - ], - ] - ); - - // add pledge fields only if its is enabled - if (CRM_Core_Permission::access('CiviPledge')) { - $pledgeFields = [ - 'pledge_payment' => [ - 'title' => ts('Pledge Payment'), - 'headerPattern' => '/Pledge Payment/i', - ], - 'pledge_id' => [ - 'title' => ts('Pledge ID'), - 'headerPattern' => '/Pledge ID/i', - ], - ]; - - $fields = array_merge($fields, $pledgeFields); - } - foreach ($fields as $name => $field) { - $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT); - $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//'); - $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//'); + $this->setFieldMetadata(); + foreach ($this->getImportableFieldsMetadata() as $name => $field) { $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']); } @@ -712,6 +683,49 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { } } + /** + * Set field metadata. + */ + protected function setFieldMetadata() { + if (empty($this->importableFieldsMetadata)) { + $fields = CRM_Contribute_BAO_Contribution::importableFields($this->_contactType, FALSE); + + $fields = array_merge($fields, + [ + 'soft_credit' => [ + 'title' => ts('Soft Credit'), + 'softCredit' => TRUE, + 'headerPattern' => '/Soft Credit/i', + ], + ] + ); + + // add pledge fields only if its is enabled + if (CRM_Core_Permission::access('CiviPledge')) { + $pledgeFields = [ + 'pledge_payment' => [ + 'title' => ts('Pledge Payment'), + 'headerPattern' => '/Pledge Payment/i', + ], + 'pledge_id' => [ + 'title' => ts('Pledge ID'), + 'headerPattern' => '/Pledge ID/i', + ], + ]; + + $fields = array_merge($fields, $pledgeFields); + } + foreach ($fields as $name => $field) { + $fields[$name] = array_merge([ + 'type' => CRM_Utils_Type::T_INT, + 'dataPattern' => '//', + 'headerPattern' => '//', + ], $field); + } + $this->importableFieldsMetadata = $fields; + } + } + /** * Handle the values in summary mode. * @@ -1553,4 +1567,35 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { return NULL; } + /** + * Get the civicrm_mapping_field appropriate layout for the mapper input. + * + * The input looks something like ['street_address', 1] + * and would be mapped to ['name' => 'street_address', 'location_type_id' => + * 1] + * + * @param array $fieldMapping + * @param int $mappingID + * @param int $columnNumber + * + * @return array + * @throws \API_Exception + */ + public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array { + $isRelationshipField = preg_match('/\d*_a_b|b_a$/', $fieldMapping[0]); + $fieldName = $isRelationshipField ? $fieldMapping[1] : $fieldMapping[0]; + return [ + 'name' => $fieldMapping[0], + 'mapping_id' => $mappingID, + 'column_number' => $columnNumber, + // The name of the field to match the soft credit on is (crazily) + // stored in 'contact_type' + 'contact_type' => $fieldMapping[1] ?? NULL, + // We also store the field in a sensible key, even if it isn't saved sensibly. + 'soft_credit_match_field' => $fieldMapping[1] ?? NULL, + // This field is actually not saved at all :-( It is lost each time. + 'soft_credit_type_id' => $fieldMapping[2] ?? NULL, + ]; + } + } diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index 083f4aceb7..9c10336654 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -193,4 +193,44 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { return ''; } + /** + * Get the field mapped to the savable format. + * + * @param array $fieldMapping + * @param int $mappingID + * @param int $columnNumber + * + * @return array + * @throws \CRM_Core_Exception + */ + protected function getMappedField(array $fieldMapping, int $mappingID, int $columnNumber): array { + return $this->getParser()->getMappingFieldFromMapperInput($fieldMapping, $mappingID, $columnNumber); + } + + /** + * Save the mapping field. + * + * @param int $mappingID + * @param int $columnNumber + * @param bool $isUpdate + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + protected function saveMappingField(int $mappingID, int $columnNumber, bool $isUpdate = FALSE): void { + $fieldMapping = (array) $this->getSubmittedValue('mapper')[$columnNumber]; + $mappedField = $this->getMappedField($fieldMapping, $mappingID, $columnNumber); + if ($isUpdate) { + Civi\Api4\MappingField::update(FALSE) + ->setValues($mappedField) + ->addWhere('column_number', '=', $columnNumber) + ->addWhere('mapping_id', '=', $mappingID) + ->execute(); + } + else { + Civi\Api4\MappingField::create(FALSE) + ->setValues($mappedField)->execute(); + } + } + } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 76d9868a35..e752dc6c83 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -550,9 +550,16 @@ class CRM_Import_Forms extends CRM_Core_Form { * @throws \API_Exception */ protected function getAvailableFields(): array { - $parser = new CRM_Contact_Import_Parser_Contact(); - $parser->setUserJobID($this->getUserJobID()); - return $parser->getAvailableFields(); + return $this->getParser()->getAvailableFields(); + } + + /** + * Get an instance of the parser class. + * + * @return \CRM_Contact_Import_Parser_Contact|\CRM_Contribute_Import_Parser_Contribution + */ + protected function getParser() { + return NULL; } } diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 7242786a56..c8651ea4d2 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -261,6 +261,45 @@ abstract class CRM_Import_Parser { $this->importableFieldsMetadata = $importableFieldsMetadata; } + /** + * Gets the fields available for importing in a key-name, title format. + * + * @return array + * eg. ['first_name' => 'First Name'.....] + * + * @throws \API_Exception + * + * @todo - we are constructing the metadata before we + * have set the contact type so we re-do it here. + * + * Once we have cleaned up the way the mapper is handled + * we can ditch all the existing _construct parameters in favour + * of just the userJobID - there are current open PRs towards this end. + */ + public function getAvailableFields(): array { + $this->setFieldMetadata(); + $return = []; + foreach ($this->getImportableFieldsMetadata() as $name => $field) { + if ($name === 'id' && $this->isSkipDuplicates()) { + // Duplicates are being skipped so id matching is not availble. + continue; + } + $return[$name] = $field['title']; + } + return $return; + } + + /** + * Did the user specify duplicates should be skipped and not imported. + * + * @return bool + * + * @throws \API_Exception + */ + protected function isSkipDuplicates(): bool { + return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP; + } + /** * Array of the fields that are actually part of the import process * the position in the array also dictates their position in the import diff --git a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php index 137625932a..836ff3faca 100644 --- a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php @@ -9,6 +9,8 @@ +--------------------------------------------------------------------+ */ +use Civi\Api4\MappingField; + /** * Upgrade logic for the 5.51.x series. * @@ -29,6 +31,44 @@ class CRM_Upgrade_Incremental_php_FiveFiftyOne extends CRM_Upgrade_Incremental_B */ public function upgrade_5_51_alpha1($rev): void { $this->addTask(ts('Upgrade DB to %1: SQL', [1 => $rev]), 'runSql', $rev); + $this->addTask(ts('Convert import mappings to use names'), 'convertMappingFieldLabelsToNames', $rev); + } + + /** + * Convert saved mapping fields for contribution imports to use name rather than + * label. + * + * Currently the 'name' column in civicrm_mapping_field holds names like + * 'First Name' or, more tragically 'Contact ID (match to contact)'. + * + * This updates them to hold the name - eg. 'total_amount'. + * + * @return bool + * @throws \API_Exception + */ + public static function convertMappingFieldLabelsToNames(): bool { + $mappings = MappingField::get(FALSE) + ->setSelect(['id', 'name']) + ->addWhere('mapping_id.mapping_type_id:name', '=', 'Import Contribution') + ->execute(); + $fields = CRM_Contribute_BAO_Contribution::importableFields('All', FALSE); + $fieldMap = []; + foreach ($fields as $fieldName => $field) { + $fieldMap[$field['title']] = $fieldName; + } + $fieldMap[ts('Soft Credit')] = 'soft_credit'; + $fieldMap[ts('Pledge Payment')] = 'pledge_payment'; + $fieldMap[ts(ts('Pledge ID'))] = 'pledge_id'; + + foreach ($mappings as $mapping) { + if (!empty($fieldMap[$mapping['name']])) { + MappingField::update(FALSE) + ->addWhere('id', '=', $mapping['id']) + ->addValue('name', $fieldMap[$mapping['name']]) + ->execute(); + } + } + return TRUE; } } -- 2.25.1