From 8daa4f362f8b1d0a3a79f23827b7740786d1822e Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Fri, 3 Jun 2022 17:12:05 +1200 Subject: [PATCH] Cleanup templates & form variables, following contact pattern --- CRM/Contribute/Import/Form/MapField.php | 37 ++---- CRM/Contribute/Import/Form/Preview.php | 32 +---- CRM/Contribute/Import/Parser/Contribution.php | 113 +++++++----------- .../CRM/Contribute/Import/Form/MapTable.tpl | 93 +------------- .../CRM/Contribute/Import/Form/Preview.tpl | 2 +- 5 files changed, 61 insertions(+), 216 deletions(-) diff --git a/CRM/Contribute/Import/Form/MapField.php b/CRM/Contribute/Import/Form/MapField.php index 8c24947a2b..b45767d8b1 100644 --- a/CRM/Contribute/Import/Form/MapField.php +++ b/CRM/Contribute/Import/Form/MapField.php @@ -72,27 +72,15 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { * Set variables up before form is built. */ public function preProcess() { + parent::preProcess(); $this->_mapperFields = $this->getAvailableFields(); asort($this->_mapperFields); $this->_columnCount = $this->get('columnCount'); - $this->assign('columnCount', $this->_columnCount); - $this->_dataValues = $this->get('dataValues'); - $this->assign('dataValues', $this->_dataValues); + $skipColumnHeader = $this->getSubmittedValue('skipColumnHeader'); + $this->_onDuplicate = $this->getSubmittedValue('onDuplicate'); + $this->assign('skipColumnHeader', $skipColumnHeader); - $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); - $this->_onDuplicate = $this->get('onDuplicate', $onDuplicate ?? ""); - - if ($skipColumnHeader) { - $this->assign('skipColumnHeader', $skipColumnHeader); - $this->assign('rowDisplayCount', 3); - // If we had a column header to skip, stash it for later - - $this->_columnHeaders = $this->_dataValues[0]; - } - else { - $this->assign('rowDisplayCount', 2); - } $highlightedFields = ['financial_type_id', 'total_amount']; //CRM-2219 removing other required fields since for updation only //invoice id or trxn id or contribution id is required. @@ -157,10 +145,11 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $defaults = []; $mapperKeys = array_keys($this->_mapperFields); - $hasHeaders = !empty($this->_columnHeaders); - $headerPatterns = $this->get('headerPatterns'); - $dataPatterns = $this->get('dataPatterns'); - $mapperKeysValues = $this->controller->exportValue($this->_name, 'mapper'); + $hasHeaders = $this->getSubmittedValue('skipColumnHeader'); + $headerPatterns = $this->getHeaderPatterns(); + $dataPatterns = $this->getDataPatterns(); + $mapperKeysValues = $this->getSubmittedValue('mapper'); + $columnHeaders = $this->getColumnHeaders(); /* Initialize all field usages to false */ foreach ($mapperKeys as $key) { @@ -188,7 +177,7 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { //used to warn for mismatch column count or mismatch mapping $warning = 0; - for ($i = 0; $i < $this->_columnCount; $i++) { + foreach ($columnHeaders as $i => $columnHeader) { $sel = &$this->addElement('hierselect', "mapper[$i]", ts('Mapper for Field %1', [1 => $i]), NULL); $jsSet = FALSE; if ($this->get('savedMapping')) { @@ -228,7 +217,7 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $js .= "swapOptions($formName, 'mapper[$i]', 0, 3, 'hs_mapper_0_');\n"; if ($hasHeaders) { - $defaults["mapper[$i]"] = [$this->defaultFromHeader($this->_columnHeaders[$i], $headerPatterns)]; + $defaults["mapper[$i]"] = [$this->defaultFromHeader($columnHeader, $headerPatterns)]; } else { $defaults["mapper[$i]"] = [$this->defaultFromData($dataPatterns, $i)]; @@ -240,7 +229,7 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { $js .= "swapOptions($formName, 'mapper[$i]', 0, 3, 'hs_mapper_0_');\n"; if ($hasHeaders) { // do array search first to see if has mapped key - $columnKey = array_search($this->_columnHeaders[$i], $this->_mapperFields); + $columnKey = array_search($columnHeader, $this->_mapperFields); if (isset($this->_fieldUsed[$columnKey])) { $defaults["mapper[$i]"] = $columnKey; $this->_fieldUsed[$key] = TRUE; @@ -248,7 +237,7 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { else { // Infer the default from the column names if we have them $defaults["mapper[$i]"] = [ - $this->defaultFromHeader($this->_columnHeaders[$i], $headerPatterns), + $this->defaultFromHeader($columnHeader, $headerPatterns), 0, ]; } diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index 46d77feedb..bce405c8e8 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -25,38 +25,16 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { */ public function preProcess() { parent::preProcess(); - //get the data from the session - $dataValues = $this->get('dataValues'); - $invalidRowCount = $this->get('invalidRowCount'); - - //get the mapping name displayed if the mappingId is set - $mappingId = $this->get('loadMappingId'); - if ($mappingId) { - $mapDAO = new CRM_Core_DAO_Mapping(); - $mapDAO->id = $mappingId; - $mapDAO->find(TRUE); - } - $this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL); + $invalidRowCount = $this->getRowCount(CRM_Import_Parser::VALID); + $downloadURL = ''; if ($invalidRowCount) { $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + $downloadURL = CRM_Utils_System::url('civicrm/export', $urlParams); } - $properties = [ - 'dataValues', - 'columnCount', - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - ]; $this->setStatusUrl(); - $this->assign('mapper', $this->getMappedFieldLabels()); - - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } + $this->assign('downloadErrorRecordsUrl', $downloadURL); } /** @@ -105,7 +83,7 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { $mapperFields, $this->getSubmittedValue('skipColumnHeader'), CRM_Import_Parser::MODE_IMPORT, - $this->get('contactType'), + $this->getSubmittedValue('contactType'), $onDuplicate, $this->get('statusID'), $this->get('totalRowCount') diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index c818b09475..239180da65 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -48,17 +48,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { */ const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096; - /** - * @var string - */ - protected $_fileName; - - /** - * Imported file size - * @var int - */ - protected $_fileSize; - /** * Separator being used * @var string @@ -137,7 +126,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * @param int $contactType * @param int $onDuplicate * @param int $statusID - * @param int $totalRowCount * * @return mixed * @throws Exception @@ -150,13 +138,8 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $mode = self::MODE_PREVIEW, $contactType = self::CONTACT_INDIVIDUAL, $onDuplicate = self::DUPLICATE_SKIP, - $statusID = NULL, - $totalRowCount = NULL + $statusID = NULL ) { - if (!is_array($fileName)) { - throw new CRM_Core_Exception('Unable to determine import file'); - } - $fileName = $fileName['name']; // Since $this->_contactType is still being called directly do a get call // here to make sure it is instantiated. $this->getContactType(); @@ -165,13 +148,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $this->_haveColumnHeader = $skipColumnHeader; - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - $this->_lineCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; $this->_totalCount = 0; @@ -185,8 +161,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $startTimestamp = $currTimestamp = $prevTimestamp = time(); } - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - if ($mode == self::MODE_MAPFIELD) { $this->_rows = []; } @@ -194,32 +168,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $this->_activeFieldCount = count($this->_activeFields); } - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } + $dataSource = $this->getDataSourceObject(); + $totalRowCount = $dataSource->getRowCount(['new']); + $dataSource->setStatuses(['new']); - if (CRM_Utils_System::isNull($values)) { - continue; - } + while ($row = $dataSource->getRow()) { + $values = array_values($row); + $this->_lineCount++; $this->_totalCount++; @@ -309,15 +264,8 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $this->_validCount++; } } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } } - fclose($fd); - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { $customHeaders = $mapper; @@ -470,18 +418,10 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * @param int $mode */ public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - $store->set('totalRowCount', $this->_totalCount); $store->set('validRowCount', $this->_validCount); $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); @@ -692,6 +632,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * CRM_Import_Parser::VALID or CRM_Import_Parser::ERROR */ public function summary(&$values) { + $rowNumber = (int) ($values[array_key_last($values)]); $params = $this->getMappedRow($values); //for date-Formats @@ -707,6 +648,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $tempMsg = "Invalid value for field(s) : $errorMessage"; array_unshift($values, $tempMsg); $errorMessage = NULL; + $this->setImportStatus($rowNumber, 'ERROR', $tempMsg); return CRM_Import_Parser::ERROR; } @@ -732,6 +674,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * - CRM_Import_Parser::PLEDGE_PAYMENT (successful creation) */ public function import($onDuplicate, &$values) { + $rowNumber = (int) ($values[array_key_last($values)]); // first make sure this is a valid line $response = $this->summary($values); if ($response != CRM_Import_Parser::VALID) { @@ -779,6 +722,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { catch (CRM_Core_Exception $e) { array_unshift($values, $e->getMessage()); $errorMapping = ['soft_credit' => self::SOFT_CREDIT_ERROR, 'pledge_payment' => self::PLEDGE_PAYMENT_ERROR]; + $this->setImportStatus($rowNumber, $errorMapping[$e->getErrorCode()] ?? CRM_Import_Parser::ERROR, $e->getMessage()); return $errorMapping[$e->getErrorCode()] ?? CRM_Import_Parser::ERROR; } @@ -790,6 +734,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') { return self::PLEDGE_PAYMENT_ERROR; } + $this->setImportStatus($rowNumber, 'ERROR', ''); return CRM_Import_Parser::ERROR; } @@ -873,6 +818,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { } $errorMsg = implode(' AND ', $errorMsg); array_unshift($values, 'Matching Contribution record not found for ' . $errorMsg . '. Row was skipped.'); + $this->setImportStatus($rowNumber, 'ERROR', 'Matching Contribution record not found for ' . $errorMsg . '. Row was skipped.'); return CRM_Import_Parser::ERROR; } } @@ -885,6 +831,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $matchedIDs = explode(',', $error['error_message']['params'][0]); if (count($matchedIDs) > 1) { array_unshift($values, 'Multiple matching contact records detected for this row. The contribution was not imported'); + $this->setImportStatus($rowNumber, 'ERROR', 'Multiple matching contact records detected for this row. The contribution was not imported'); return CRM_Import_Parser::ERROR; } $cid = $matchedIDs[0]; @@ -895,11 +842,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { if (is_array($newContribution['error_message'])) { array_unshift($values, $newContribution['error_message']['message']); if ($newContribution['error_message']['params'][0]) { + $this->setImportStatus($rowNumber, 'DUPLICATE', $newContribution['error_message']['message']); return CRM_Import_Parser::DUPLICATE; } } else { array_unshift($values, $newContribution['error_message']); + $this->setImportStatus($rowNumber, 'ERROR', $newContribution['error_message']); return CRM_Import_Parser::ERROR; } } @@ -943,8 +892,9 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $disp = $params['external_identifier']; } } - - array_unshift($values, 'No matching Contact found for (' . $disp . ')'); + $errorMessage = 'No matching Contact found for (' . $disp . ')'; + $this->setImportStatus($rowNumber, 'ERROR', $errorMessage); + array_unshift($values, $errorMessage); return CRM_Import_Parser::ERROR; } @@ -953,7 +903,9 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $checkCid->external_identifier = $paramValues['external_identifier']; $checkCid->find(TRUE); if ($checkCid->id != $formatted['contact_id']) { - array_unshift($values, 'Mismatch of External ID:' . $paramValues['external_identifier'] . ' and Contact Id:' . $formatted['contact_id']); + $errorMessage = 'Mismatch of External ID:' . $paramValues['external_identifier'] . ' and Contact Id:' . $formatted['contact_id']; + array_unshift($values, $errorMessage); + $this->setImportStatus($rowNumber, 'ERROR', $errorMessage); return CRM_Import_Parser::ERROR; } } @@ -962,11 +914,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { if (is_array($newContribution['error_message'])) { array_unshift($values, $newContribution['error_message']['message']); if ($newContribution['error_message']['params'][0]) { + $this->setImportStatus($rowNumber, 'DUPLICATE', ''); return CRM_Import_Parser::DUPLICATE; } } else { array_unshift($values, $newContribution['error_message']); + $this->setImportStatus($rowNumber, 'ERROR', $newContribution['error_message']); return CRM_Import_Parser::ERROR; } } @@ -1509,7 +1463,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $lookupField = !empty($params['contact_id']) ? 'contact_id' : (!empty($params['external_identifier']) ? 'external_identifier' : 'email'); if (empty($params['email'])) { $contact = Contact::get(FALSE)->addSelect('id') - ->addWhere($lookupField, '=', $params[$lookupField]) + ->addWhere($lookupField === 'contact_id' ? 'id' : $lookupField, '=', $params[$lookupField]) ->execute(); if (count($contact) !== 1) { throw new CRM_Core_Exception(ts("Soft Credit %1 - %2 doesn't exist. Row was skipped.", @@ -1562,4 +1516,19 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { return implode(' - ', $title); } + /** + * Get the metadata field for which importable fields does not key the actual field name. + * + * @return string[] + */ + protected function getOddlyMappedMetadataFields(): array { + $uniqueNames = ['contribution_id', 'contribution_contact_id', 'contribution_cancel_date', 'contribution_source', 'contribution_check_number']; + $fields = []; + foreach ($uniqueNames as $name) { + $fields[$this->importableFieldsMetadata[$name]['name']] = $name; + } + // Include the parent fields as they could be present if required for matching ...in theory. + return array_merge($fields, parent::getOddlyMappedMetadataFields()); + } + } diff --git a/templates/CRM/Contribute/Import/Form/MapTable.tpl b/templates/CRM/Contribute/Import/Form/MapTable.tpl index 458bf3b4fe..3b9bab3228 100644 --- a/templates/CRM/Contribute/Import/Form/MapTable.tpl +++ b/templates/CRM/Contribute/Import/Form/MapTable.tpl @@ -8,95 +8,4 @@ +--------------------------------------------------------------------+ *} {* Contribution Import Wizard - Data Mapping table used by MapFields.tpl and Preview.tpl *} - -
- {strip} - - {if $savedMappingName} - - {/if} - - {section name=rows loop=$rowDisplayCount} - {if $skipColumnHeader } - {if $smarty.section.rows.iteration == 1} - - {else} - - {/if} - {else} - - {/if} - {/section} - - - - - {*Loop on columns parsed from the import data rows*} - {section name=cols loop=$columnCount} - {assign var="i" value=$smarty.section.cols.index} - - - {section name=rows loop=$rowDisplayCount} - {assign var="j" value=$smarty.section.rows.index} - - {/section} - - {* Display mapper - - - {/section} - -
{ts 1=$savedMappingName}Saved Field Mapping: %1{/ts}
{ts}Column Headers{/ts}{ts 1=$smarty.section.rows.iteration}Import Data (row %1){/ts}{ts 1=$smarty.section.rows.iteration}Import Data (row %1){/ts}{ts}Matching CiviCRM Field{/ts}
{$dataValues[$j][$i]|escape} - {if $wizard.currentStepName == 'Preview'} - {$mapper[$i]} - {else} - {$form.mapper[$i].html|smarty:nodefaults} - {/if} -
- {/strip} - - {if $wizard.currentStepName != 'Preview'} -
- - {if $savedMappingName} - {$form.updateMapping.html}    {$form.updateMapping.label} - {/if} - {$form.saveMapping.html}    {$form.saveMapping.label} -
- - - - - - - - - -
{$form.saveMappingName.label}{$form.saveMappingName.html}
{$form.saveMappingDesc.label}{$form.saveMappingDesc.html}
-
- -
- {/if} -
+{include file="CRM/Import/Form/MapTableCommon.tpl" mapper=$form.mapper} diff --git a/templates/CRM/Contribute/Import/Form/Preview.tpl b/templates/CRM/Contribute/Import/Form/Preview.tpl index bf4557bcf0..61f1d14e11 100644 --- a/templates/CRM/Contribute/Import/Form/Preview.tpl +++ b/templates/CRM/Contribute/Import/Form/Preview.tpl @@ -54,6 +54,6 @@ {* Table for mapping preview *} - {include file="CRM/Contribute/Import/Form/MapTable.tpl"} +{include file="CRM/Import/Form/MapTableCommon.tpl"}
{include file="CRM/common/formButtons.tpl" location="bottom"}
-- 2.25.1