From b7dde70cb8586527cda1b12c816eca40ba240239 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Wed, 8 Jun 2022 12:50:50 +1200 Subject: [PATCH] Fix Contribution import with pledge handling, add test --- CRM/Contribute/Import/Form/MapField.php | 47 +--- CRM/Contribute/Import/Form/Preview.php | 60 ----- CRM/Contribute/Import/Parser/Contribution.php | 236 +++++------------- CRM/Import/Parser.php | 8 +- .../Import/Parser/ContributionTest.php | 24 ++ .../Contribute/Import/Parser/data/pledge.csv | 2 +- 6 files changed, 90 insertions(+), 287 deletions(-) diff --git a/CRM/Contribute/Import/Form/MapField.php b/CRM/Contribute/Import/Form/MapField.php index abd920c186..9026f66c5c 100644 --- a/CRM/Contribute/Import/Form/MapField.php +++ b/CRM/Contribute/Import/Form/MapField.php @@ -376,49 +376,12 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField { } /** - * Process the mapped fields and map it into the uploaded file preview the file and extract some summary statistics. + * Get the mapping name per the civicrm_mapping_field.type_id option group. + * + * @return string */ - public function postProcess() { - $params = $this->controller->exportValues('MapField'); - - //reload the mapfield if load mapping is pressed - if (!empty($params['savedMapping'])) { - $this->set('savedMapping', $params['savedMapping']); - $this->controller->resetPage($this->_name); - return; - } - $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); - - $mapper = $mapperKeysMain = $mapperSoftCredit = $softCreditFields = $mapperPhoneType = $mapperSoftCreditType = []; - $mapperKeys = $this->controller->exportValue($this->_name, 'mapper'); - - $softCreditTypes = CRM_Core_OptionGroup::values('soft_credit_type'); - - for ($i = 0; $i < $this->_columnCount; $i++) { - $mapper[$i] = $this->_mapperFields[$mapperKeys[$i][0]]; - $mapperKeysMain[$i] = $mapperKeys[$i][0]; - } - - $this->set('mapper', $mapper); - - // store mapping Id to display it in the preview page - $this->set('loadMappingId', CRM_Utils_Array::value('mappingId', $params)); - - $mappingType = 'Import Contribution'; - $this->saveMapping($mappingType); - - $parser = new CRM_Contribute_Import_Parser_Contribution($mapperKeysMain); - $parser->setUserJobID($this->getUserJobID()); - $parser->run( - $this->getSubmittedValue('uploadFile'), - $this->getSubmittedValue('fieldSeparator'), - $mapper, - $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_PREVIEW - ); - - // add all the necessary variables to the form - $parser->set($this); + public function getMappingTypeName(): string { + return 'Import Contribution'; } /** diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index bce405c8e8..3bfc113f7b 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -56,66 +56,6 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { return $mapper; } - /** - * Process the mapped fields and map it into the uploaded file preview the file and extract some summary statistics. - */ - public function postProcess() { - $fileName = $this->controller->exportValue('DataSource', 'uploadFile'); - $onDuplicate = $this->get('onDuplicate'); - $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); - $mapper = $this->controller->exportValue('MapField', 'mapper'); - - $parser = new CRM_Contribute_Import_Parser_Contribution(); - $parser->setUserJobID($this->getUserJobID()); - - $mapFields = $this->get('fields'); - - foreach ($mapper as $key => $value) { - $header = []; - if (isset($mapFields[$mapper[$key][0]])) { - $header[] = $mapFields[$mapper[$key][0]]; - } - $mapperFields[] = implode(' - ', $header); - } - $parser->run( - $this->getSubmittedValue('uploadFile'), - $this->getSubmittedValue('fieldSeparator'), - $mapperFields, - $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_IMPORT, - $this->getSubmittedValue('contactType'), - $onDuplicate, - $this->get('statusID'), - $this->get('totalRowCount') - ); - - // Add all the necessary variables to the form. - $parser->set($this, CRM_Import_Parser::MODE_IMPORT); - - // Check if there is any error occurred. - - $errorStack = CRM_Core_Error::singleton(); - $errors = $errorStack->getErrors(); - $errorMessage = []; - - if (is_array($errors)) { - foreach ($errors as $key => $value) { - $errorMessage[] = $value['message']; - } - - $errorFile = $fileName['name'] . '.error.log'; - - if ($fd = fopen($errorFile, 'w')) { - fwrite($fd, implode('\n', $errorMessage)); - } - fclose($fd); - - $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - } - /** * @return \CRM_Contribute_Import_Parser_Contribution */ diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index 3e087ad68e..c69cd37002 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -177,141 +177,14 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $returnCode = $this->summary($values); } elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($values); + $this->import($values); if ($statusID && (($this->_lineCount % 50) == 0)) { $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); } } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode == self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::SOFT_CREDIT) { - $this->_validSoftCreditRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::PLEDGE_PAYMENT) { - $this->_validPledgePaymentRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - if ($returnCode == self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { - $this->_invalidPledgePaymentRowCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_pledgePaymentErrors[] = $values; - } - - if ($returnCode == self::SOFT_CREDIT_ERROR) { - $this->_invalidSoftCreditRowCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_softCreditErrors[] = $values; - } - - if ($returnCode == self::DUPLICATE) { - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if (!$this->isSkipDuplicates()) { - $this->_validCount++; - } - } } - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - - if ($this->_invalidPledgePaymentRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); - self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); - } - - if ($this->_invalidSoftCreditRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); - self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); - } - - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Contribution URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } - } - } - - /** - * Given a list of the importable field keys that the user has selected - * set the active fields array to this list - * - * @param array $fieldKeys mapped array of values - */ - public function setActiveFields($fieldKeys) { - $this->_activeFieldCount = count($fieldKeys); - foreach ($fieldKeys as $key) { - if (empty($this->_fields[$key])) { - $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); - } - else { - $this->_activeFields[] = clone($this->_fields[$key]); - } - } } /** @@ -335,6 +208,15 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { return $mappedFields; } + /** + * Get the required fields. + * + * @return array + */ + public function getRequiredFields(): array { + return ['id' => ts('Contribution ID'), ['financial_type_id' => ts('Financial Type'), 'total_amount' => ts('Total Amount')]]; + } + /** * Transform the input parameters into the form handled by the input routine. * @@ -555,10 +437,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { foreach ($this->getImportableFieldsMetadata() as $name => $field) { $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']); } - - $this->_newContributions = []; - - $this->setActiveFields($this->_mapperKeys); } /** @@ -581,13 +459,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { // add pledge fields only if its is enabled if (CRM_Core_Permission::access('CiviPledge')) { $pledgeFields = [ - 'pledge_payment' => [ - 'title' => ts('Pledge Payment'), - 'headerPattern' => '/Pledge Payment/i', - ], 'pledge_id' => [ 'title' => ts('Pledge ID'), 'headerPattern' => '/Pledge ID/i', + 'name' => 'pledge_id', + 'entity' => 'Pledge', + 'type' => CRM_Utils_Type::T_INT, + 'options' => FALSE, ], ]; @@ -620,9 +498,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $params['contact_type'] = 'Contribution'; if ($errorMessage) { - $tempMsg = "Invalid value for field(s) : $errorMessage"; - array_unshift($values, $tempMsg); - $this->setImportStatus($rowNumber, 'ERROR', $tempMsg); + $this->setImportStatus($rowNumber, 'ERROR', "Invalid value for field(s) : $errorMessage"); return CRM_Import_Parser::ERROR; } @@ -634,18 +510,8 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { * * @param array $values * The array of values belonging to this line. - * - * @return int - * the result of this processing - one of - * - CRM_Import_Parser::VALID - * - CRM_Import_Parser::ERROR - * - CRM_Import_Parser::SOFT_CREDIT_ERROR - * - CRM_Import_Parser::PLEDGE_PAYMENT_ERROR - * - CRM_Import_Parser::DUPLICATE - * - CRM_Import_Parser::SOFT_CREDIT (successful creation) - * - CRM_Import_Parser::PLEDGE_PAYMENT (successful creation) */ - public function import(&$values) { + public function import($values): void { $rowNumber = (int) ($values[array_key_last($values)]); try { $params = $this->getMappedRow($values); @@ -682,12 +548,11 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { $formatError = $this->deprecatedFormatParams($paramValues, $formatted); if ($formatError) { - array_unshift($values, $formatError['error_message']); if (CRM_Utils_Array::value('error_data', $formatError) == 'soft_credit') { - return self::SOFT_CREDIT_ERROR; + throw new CRM_Core_Exception('', self::SOFT_CREDIT_ERROR); } if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') { - return self::PLEDGE_PAYMENT_ERROR; + throw new CRM_Core_Exception('', self::PLEDGE_PAYMENT_ERROR); } throw new CRM_Core_Exception('', CRM_Import_Parser::ERROR); } @@ -754,11 +619,14 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return self::SOFT_CREDIT; + $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT)); + return; } // process pledge payment assoc w/ the contribution - return $this->processPledgePayments($formatted); + $this->processPledgePayments($formatted); + $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT)); + return; } $labels = [ 'id' => 'Contribution ID', @@ -804,11 +672,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return self::SOFT_CREDIT; + $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT)); + return; } - // process pledge payment assoc w/ the contribution - return $this->processPledgePayments($formatted); + $this->processPledgePayments($formatted); + $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT)); + return; } // Using new Dedupe rule. @@ -868,26 +738,44 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return self::SOFT_CREDIT; + $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT), ''); + return; } // process pledge payment assoc w/ the contribution - return $this->processPledgePayments($formatted); + $this->processPledgePayments($formatted); + $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT)); + return; + } catch (CRM_Core_Exception $e) { - array_unshift($values, $e->getMessage()); - $errorMapping = [self::SOFT_CREDIT_ERROR => 'soft_credit_error', self::PLEDGE_PAYMENT_ERROR => 'pledge_payment_error', CRM_Import_Parser::DUPLICATE => 'DUPLICATE']; - $this->setImportStatus($rowNumber, $errorMapping[$e->getErrorCode()] ?? 'ERROR', $e->getMessage()); - return $errorMapping[$e->getErrorCode()] ?? CRM_Import_Parser::ERROR; + $this->setImportStatus($rowNumber, $this->getStatus($e->getErrorCode()), $e->getMessage()); } } + /** + * Get the status to record. + * + * @param int|null $code + * + * @return string + */ + protected function getStatus(?int $code): string { + $errorMapping = [ + self::SOFT_CREDIT_ERROR => 'soft_credit_error', + self::PLEDGE_PAYMENT_ERROR => 'pledge_payment_error', + self::SOFT_CREDIT => 'soft_credit_imported', + self::PLEDGE_PAYMENT => 'pledge_payment_imported', + CRM_Import_Parser::DUPLICATE => 'DUPLICATE', + CRM_Import_Parser::VALID => 'IMPORTED', + ]; + return $errorMapping[$code] ?? 'ERROR'; + } + /** * Process pledge payments. * * @param array $formatted - * - * @return int */ private function processPledgePayments(array $formatted) { if (!empty($formatted['pledge_payment_id']) && !empty($formatted['pledge_id'])) { @@ -905,8 +793,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { NULL, $formatted['total_amount'] ); - - return self::PLEDGE_PAYMENT; } } @@ -1069,25 +955,15 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { } break; - case 'pledge_payment': case 'pledge_id': - - // giving respect to pledge_payment flag. - if (empty($params['pledge_payment'])) { - break; - } - // get total amount of from import fields $totalAmount = $params['total_amount'] ?? NULL; - - $onDuplicate = $params['onDuplicate'] ?? NULL; - // we need to get contact id $contributionContactID to // retrieve pledge details as well as to validate pledge ID // first need to check for update mode if ($this->isUpdateExisting() && - ($params['contribution_id'] || $params['trxn_id'] || $params['invoice_id']) + ($params['id'] || $params['trxn_id'] || $params['invoice_id']) ) { $contribution = new CRM_Contribute_DAO_Contribution(); if ($params['contribution_id']) { @@ -1107,7 +983,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { } } else { - return civicrm_api3_create_error('No match found for specified contact in pledge payment data. Row was skipped.'); + throw new CRM_Core_Exception('No match found for specified contact in pledge payment data. Row was skipped.'); } } else { diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index bd0a0d9e1f..7295afdc07 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -1630,7 +1630,9 @@ abstract class CRM_Import_Parser { * @throws \CRM_Core_Exception */ protected function validateParams(array $params): void { - $this->validateRequiredFields($this->getRequiredFields(), $params); + if (empty($params['id'])) { + $this->validateRequiredFields($this->getRequiredFields(), $params); + } $errors = []; foreach ($params as $key => $value) { $errors = array_merge($this->getInvalidValues($value, $key), $errors); @@ -1941,10 +1943,8 @@ abstract class CRM_Import_Parser { * * @noinspection PhpDocMissingThrowsInspection * @noinspection PhpUnhandledExceptionInspection - * @throws \API_Exception - * @throws \CRM_Core_Exception */ - protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, $additionalFields = []): void { + protected function setImportStatus(int $id, string $status, string $message = '', ?int $entityID = NULL, $additionalFields = []): void { $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields); } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index 49c3282e8f..bc8d41ad57 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -88,6 +88,10 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $contributionsOfSoftContact = ContributionSoft::get()->addWhere('contact_id', '=', $contact2Id)->execute(); $this->assertCount(1, $contributionsOfSoftContact, 'Contribution Soft not added for primary contact'); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::ERROR])); + $this->assertEquals(1, $dataSource->getRowCount([CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT])); + $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::VALID])); } /** @@ -174,6 +178,26 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { $this->callAPISuccess('CustomGroup', 'delete', ['id' => $this->ids['CustomGroup']['Custom Group']]); } + /** + * Test importing to a pledge. + */ + public function testPledgeImport(): void { + $contactID = $this->individualCreate(['email' => 'mum@example.com']); + $pledgeID = $this->pledgeCreate(['contact_id' => $contactID]); + $this->importCSV('pledge.csv', [ + ['name' => 'email'], + ['name' => 'total_amount'], + ['name' => 'pledge_id'], + ['name' => 'receive_date'], + ['name' => 'financial_type_id'], + ], ['onDuplicate' => CRM_Import_Parser::NO_MATCH]); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $this->assertEquals(1, $dataSource->getRowCount([CRM_Contribute_Import_Parser_Contribution::PLEDGE_PAYMENT])); + $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::VALID])); + $contribution = $this->callAPISuccessGetSingle('Contribution', ['contact_id' => $contactID]); + $this->callAPISuccessGetSingle('PledgePayment', ['pledge_id' => $pledgeID, 'contribution_id' => $contribution['id']]); + } + /** * Import the csv file values. * diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/data/pledge.csv b/tests/phpunit/CRM/Contribute/Import/Parser/data/pledge.csv index 1ce47a4434..181620197a 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/data/pledge.csv +++ b/tests/phpunit/CRM/Contribute/Import/Parser/data/pledge.csv @@ -1,2 +1,2 @@ Email,Amount,Pledge ID,Receive date,Financial Type -mum@example.com,5,1,2022-01-07,Donation +mum@example.com,20,1,2022-01-07,Donation -- 2.25.1