From c4f66023e6389b134a3e11bdaf256a5bf25f935c Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Tue, 10 May 2022 20:07:14 +1200 Subject: [PATCH] Use table rather than csvs for no-match & unparsed --- CRM/Contact/Import/Form/Preview.php | 13 +- CRM/Contact/Import/Form/Summary.php | 64 ++------- CRM/Contact/Import/Parser/Contact.php | 129 +----------------- CRM/Import/DataSource.php | 2 + templates/CRM/Contact/Import/Form/Summary.tpl | 2 +- .../CRM/Contact/Import/Parser/ContactTest.php | 10 +- 6 files changed, 31 insertions(+), 189 deletions(-) diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index 931cd856ad..2fe794b233 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -34,7 +34,6 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { * @throws \CRM_Core_Exception */ public function preProcess() { - $mismatchCount = $this->get('unMatchCount'); $columnNames = $this->get('columnNames'); $this->_disableUSPS = $this->get('disableUSPS'); @@ -64,14 +63,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID)); $this->assign('totalRowCount', $this->getRowCount([])); - - if ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - $this->assign('mapper', $this->getMappedFieldLabels()); - $this->assign('dataValues', $this->getDataRows([], 2)); $this->setStatusUrl(); @@ -191,7 +183,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { 'allTags' => $this->get('tag'), 'mapper' => $this->controller->exportValue('MapField', 'mapper'), 'mapFields' => $this->getAvailableFields(), - 'contactType' => $this->get('contactType'), + 'contactType' => $this->getContactType(), 'contactSubType' => $this->getSubmittedValue('contactSubType'), 'primaryKeyName' => '_id', 'statusFieldName' => '_status', @@ -242,9 +234,6 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { fclose($fd); $this->set('errorFile', $errorFile); - - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } //hack to clean db diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php index 6ca002b3c7..d82ace038c 100644 --- a/CRM/Contact/Import/Form/Summary.php +++ b/CRM/Contact/Import/Form/Summary.php @@ -22,79 +22,43 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary { /** * Set variables up before form is built. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ public function preProcess() { // set the error message path to display $this->assign('errorFile', $this->get('errorFile')); - - $totalRowCount = $this->getRowCount(); - $relatedCount = $this->get('relatedCount'); - $totalRowCount += $relatedCount; - - $invalidRowCount = $this->get('invalidRowCount'); - $duplicateRowCount = $this->get('duplicateRowCount'); $onDuplicate = $this->get('onDuplicate'); - $mismatchCount = $this->get('unMatchCount'); - $unparsedAddressCount = $this->get('unparsedAddressCount'); - if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - elseif ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - else { - $duplicateRowCount = 0; - $this->set('duplicateRowCount', $duplicateRowCount); - } - if ($unparsedAddressCount) { - $urlParams = 'type=' . CRM_Import_Parser::UNPARSED_ADDRESS_WARNING . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->assign('downloadAddressRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $unparsedStreetAddressString = ts('Records imported successfully but unable to parse some of the street addresses'); - $this->assign('unparsedStreetAddressString', $unparsedStreetAddressString); - } $this->assign('dupeError', FALSE); if ($onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { - $dupeActionString = ts('These records have been updated with the imported data.'); + $this->assign('dupeActionString', ts('These records have been updated with the imported data.')); } elseif ($onDuplicate == CRM_Import_Parser::DUPLICATE_REPLACE) { - $dupeActionString = ts('These records have been replaced with the imported data.'); + $this->assign('dupeActionString', ts('These records have been replaced with the imported data.')); } elseif ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) { - $dupeActionString = ts('These records have been filled in with the imported data.'); + $this->assign('dupeActionString', ts('These records have been filled in with the imported data.')); } else { /* Skip by default */ - - $dupeActionString = ts('These records have not been imported.'); - + $this->assign('dupeActionString', ts('These records have not been imported.')); $this->assign('dupeError', TRUE); } - //now we also create relative contact in update and fill mode - $this->set('validRowCount', $totalRowCount - $invalidRowCount - - $duplicateRowCount - $mismatchCount - ); - $this->assign('dupeActionString', $dupeActionString); - - $properties = [ - 'downloadMismatchRecordsUrl', - 'groupAdditions', - 'tagAdditions', - 'unMatchCount', - 'unparsedAddressCount', - ]; - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } + $this->assign('groupAdditions', $this->get('groupAdditions')); + $this->assign('tagAdditions', $this->get('tagAdditions')); $this->assign('totalRowCount', $this->getRowCount()); - $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID)); + $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); $this->assign('duplicateRowCount', $this->getRowCount(CRM_Import_Parser::DUPLICATE)); + $this->assign('unMatchCount', $this->getRowCount(CRM_Import_Parser::NO_MATCH)); + $this->assign('unparsedAddressCount', $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE)); $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR)); + $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH)); + $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1')); } diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 9efce26250..262b8e283a 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -74,13 +74,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { */ protected $_newRelatedContacts; - /** - * Array of all the contacts whose street addresses are not parsed. - * of this import process - * @var array - */ - protected $_unparsedStreetAddressContacts; - protected $_tableName; /** @@ -90,33 +83,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { */ protected $_rowCount; - /** - * Running total number of un-matched Contacts. - * - * @var int - */ - protected $_unMatchCount; - - /** - * Array of unmatched lines. - * - * @var array - */ - protected $_unMatch; - - /** - * Total number of contacts with unparsed addresses - * @var int - */ - protected $_unparsedAddressCount; - - /** - * Filename of mismatch data - * - * @var string - */ - protected $_misMatchFilemName; - protected $_primaryKeyName; protected $_statusFieldName; @@ -2418,12 +2384,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $this->_errors[] = $values; } - if ($returnCode & self::NO_MATCH) { - $this->_unMatchCount++; - array_unshift($values, $this->_rowCount); - $this->_unMatch[] = $values; - } - if ($returnCode & self::DUPLICATE) { $this->_duplicateCount++; array_unshift($values, $this->_rowCount); @@ -2433,44 +2393,12 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } } - if ($returnCode & self::UNPARSED_ADDRESS_WARNING) { - $this->_unparsedAddressCount++; - array_unshift($values, $this->_rowCount); - $this->_unparsedAddresses[] = $values; - } - - // see if we've hit our timeout yet - /* if ( $the_thing_with_the_stuff ) { - do_something( ); - } */ - } - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } + if ($returnCode & self::NO_MATCH) { + $this->setImportStatus((int) $values[count($values) - 1], 'invalid_no_match', array_shift($values)); } - if ($this->_unMatchCount) { - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - - $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH); - self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch); - } - if ($this->_unparsedAddressCount) { - $headers = array_merge([ - ts('Line Number'), - ts('Contact Edit URL'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING); - self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses); + if ($returnCode & self::UNPARSED_ADDRESS_WARNING) { + $this->setImportStatus((int) $values[count($values) - 1], 'warning_unparsed_address', array_shift($values)); } } } @@ -2650,55 +2578,8 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { * @param int $mode */ public function set($store, $mode = self::MODE_SUMMARY) { - // @todo - this params are being set here because they were / possibly still - // are in some places being accessed by forms later in the flow - // ie CRM_Contact_Import_Form_MapField, CRM_Contact_Import_Form_Preview - // or CRM_Contact_Import_Form_Summary using `$this->get() - // which was the old way of saving values submitted on this form such that - // the other forms could access them. Now they should use - // `getSubmittedValue` or simply not get them if the only - // reason is to pass to the Parser which can itself - // call 'getSubmittedValue' - // Once the mentioned forms no longer call $this->get() all this 'setting' - // is obsolete. - $store->set('rowCount', $this->_rowCount); + // To be removed in https://github.com/civicrm/civicrm-core/pull/23281 $store->set('fieldTypes', $this->getSelectTypes()); - - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - $store->set('unMatchCount', $this->_unMatchCount); - - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($this->_unMatchCount) { - $store->set('mismatchFileName', $this->_misMatchFilemName); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - $store->set('unparsedAddressCount', $this->_unparsedAddressCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } } /** diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index fee6cb2dc5..1ef9a69c5e 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -471,6 +471,8 @@ abstract class CRM_Import_DataSource { CRM_Import_Parser::VALID => ['imported', 'new'], CRM_Import_Parser::ERROR => ['error', 'invalid'], CRM_Import_Parser::DUPLICATE => ['duplicate'], + CRM_Import_Parser::NO_MATCH => ['invalid_no_match'], + CRM_Import_Parser::UNPARSED_ADDRESS_WARNING => ['warning_unparsed_address'], ]; } diff --git a/templates/CRM/Contact/Import/Form/Summary.tpl b/templates/CRM/Contact/Import/Form/Summary.tpl index e3ce9d67d7..0aa01607fd 100644 --- a/templates/CRM/Contact/Import/Form/Summary.tpl +++ b/templates/CRM/Contact/Import/Form/Summary.tpl @@ -47,7 +47,7 @@ {/if} {if $unparsedAddressCount} -

{$unparsedStreetAddressString}

+

{ts}Records imported successfully but unable to parse some of the street addresses{/ts}

{ts 1=$downloadAddressRecordsUrl}You can Download Street Address Records . You may then edit those contact records and update the street address accordingly.{/ts}

diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php index 2dce16a130..268a8b046f 100644 --- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php +++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php @@ -767,8 +767,9 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { * @dataProvider importDataProvider * * @throws \API_Exception + * @throws \CRM_Core_Exception */ - public function testImport($csv, $mapper, $expectedError): void { + public function testImport($csv, $mapper, $expectedError, $expectedOutcomes = []): void { try { $this->importCSV($csv, $mapper); } @@ -779,6 +780,10 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { if ($expectedError) { $this->fail('expected error :' . $expectedError); } + $dataSource = new CRM_Import_DataSource_CSV(UserJob::get(FALSE)->setSelect(['id'])->execute()->first()['id']); + foreach ($expectedOutcomes as $outcome => $count) { + $this->assertEquals($dataSource->getRowCount([$outcome]), $count); + } } /** @@ -791,7 +796,8 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { 'individual_invalid_sub_type' => [ 'csv' => 'individual_invalid_contact_sub_type.csv', 'mapper' => [['first_name'], ['last_name'], ['contact_sub_type']], - 'expected_error' => 'Invalid value for field(s) : type', + 'expected_error' => '', + 'expected_outcomes' => [CRM_Import_Parser::NO_MATCH => 1], ], ]; } -- 2.25.1