From 99e3c5f721637dc5c43a880ae4e0f5d0f764cc53 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Sun, 24 Apr 2022 14:41:05 +1200 Subject: [PATCH] Add output form for csv-on-the-fly --- CRM/Contact/Import/Form/MapField.php | 2 +- CRM/Contact/Import/Form/Preview.php | 25 ++--- CRM/Contact/Import/Form/Summary.php | 9 +- CRM/Contact/Import/Parser/Contact.php | 25 ----- CRM/Core/xml/Menu/Import.xml | 6 ++ CRM/Import/DataSource.php | 14 +-- CRM/Import/Form/Summary.php | 2 +- CRM/Import/Forms.php | 97 ++++++++++++++++++- templates/CRM/Contact/Import/Form/Preview.tpl | 4 +- templates/CRM/Contact/Import/Form/Summary.tpl | 4 +- 10 files changed, 128 insertions(+), 60 deletions(-) diff --git a/CRM/Contact/Import/Form/MapField.php b/CRM/Contact/Import/Form/MapField.php index 1685116c53..6de8494b25 100644 --- a/CRM/Contact/Import/Form/MapField.php +++ b/CRM/Contact/Import/Form/MapField.php @@ -113,7 +113,7 @@ class CRM_Contact_Import_Form_MapField extends CRM_Import_Form_MapField { $this->assign('columnNames', $this->getColumnHeaders()); //$this->_columnCount = $this->get( 'columnCount' ); $this->assign('columnCount', $this->_columnCount); - $this->_dataValues = array_values($this->getDataRows(2)); + $this->_dataValues = array_values($this->getDataRows([], 2)); $this->assign('dataValues', $this->_dataValues); } diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index 5699d7db05..3818a1ce00 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -29,10 +29,11 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { /** * Set variables up before form is built. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ public function preProcess() { - $mapper = $this->get('mapper'); - $invalidRowCount = $this->get('invalidRowCount'); $conflictRowCount = $this->get('conflictRowCount'); $mismatchCount = $this->get('unMatchCount'); $columnNames = $this->get('columnNames'); @@ -60,11 +61,12 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { $this->set('tag', $tag); } - if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } + $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR)); + $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); + $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID)); + $this->assign('totalRowCount', $this->getRowCount([])); + // @todo conflict rows are still being output in the parser & not updating the temp table - fix if ($conflictRowCount) { $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); @@ -76,12 +78,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { } $properties = array( - 'columnCount', - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', 'conflictRowCount', - 'downloadErrorRecordsUrl', 'downloadConflictRecordsUrl', 'downloadMismatchRecordsUrl', ); @@ -91,7 +88,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { foreach ($properties as $property) { $this->assign($property, $this->get($property)); } - $this->assign('dataValues', $this->getDataRows(2)); + $this->assign('dataValues', $this->getDataRows([], 2)); $this->setStatusUrl(); } @@ -263,9 +260,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - + // @todo - these should use the new url but are not reliably updating the table yet. $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php index 4b11b5b1fb..ab526d4e01 100644 --- a/CRM/Contact/Import/Form/Summary.php +++ b/CRM/Contact/Import/Form/Summary.php @@ -27,7 +27,7 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary { // set the error message path to display $this->assign('errorFile', $this->get('errorFile')); - $totalRowCount = $this->get('totalRowCount'); + $totalRowCount = $this->getRowCount(); $relatedCount = $this->get('relatedCount'); $totalRowCount += $relatedCount; @@ -81,9 +81,6 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary { $this->assign('dupeActionString', $dupeActionString); $properties = [ - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', 'conflictRowCount', 'downloadConflictRecordsUrl', 'downloadErrorRecordsUrl', @@ -98,6 +95,10 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary { foreach ($properties as $property) { $this->assign($property, $this->get($property)); } + $this->assign('totalRowCount', $this->getRowCount()); + $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID)); + $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); + $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE)); $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1')); diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 53afad761a..4496bfc346 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -2715,15 +2715,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } if ($this->_conflictCount) { $headers = array_merge([ ts('Line Number'), @@ -2732,15 +2723,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $this->_conflictFileName = self::errorFileName(self::CONFLICT); self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); } - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Contact URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } if ($this->_unMatchCount) { $headers = array_merge([ ts('Line Number'), @@ -3064,9 +3046,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); } - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } if ($this->_conflictCount) { $store->set('conflictsFileName', $this->_conflictFileName); } @@ -3084,11 +3063,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { if ($this->_duplicateCount) { $store->set('duplicatesFileName', $this->_duplicateFileName); } - if ($this->_unparsedAddressCount) { - $store->set('errorsFileName', $this->_errorFileName); - } } - //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount"; } /** diff --git a/CRM/Core/xml/Menu/Import.xml b/CRM/Core/xml/Menu/Import.xml index e1dec69163..33d935c634 100644 --- a/CRM/Core/xml/Menu/Import.xml +++ b/CRM/Core/xml/Menu/Import.xml @@ -17,6 +17,12 @@ CRM_Contact_Import_Controller 410 + + civicrm/import/outcome + Import results + access CiviCRM + CRM_Import_Forms::outputCSV + civicrm/import/activity Import Activities diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index 77db320a37..311711e37d 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -454,13 +454,13 @@ abstract class CRM_Import_DataSource { /** * Get the mapping of constants to database status codes. * - * @return string[] + * @return array[] */ - protected function getStatusMapping() { + protected function getStatusMapping(): array { return [ - CRM_Import_Parser::VALID => 'imported', - CRM_Import_Parser::ERROR => 'error', - CRM_Import_Parser::DUPLICATE => 'duplicate', + CRM_Import_Parser::VALID => ['imported', 'new'], + CRM_Import_Parser::ERROR => ['error'], + CRM_Import_Parser::DUPLICATE => ['duplicate'], ]; } @@ -473,7 +473,9 @@ abstract class CRM_Import_DataSource { if (!empty($this->statuses)) { $statuses = []; foreach ($this->statuses as $status) { - $statuses[] = '"' . $this->getStatusMapping()[$status] . '"'; + foreach ($this->getStatusMapping()[$status] as $statusName) { + $statuses[] = '"' . $statusName . '"'; + } } return ' WHERE _status IN (' . implode(',', $statuses) . ')'; } diff --git a/CRM/Import/Form/Summary.php b/CRM/Import/Form/Summary.php index e12cbb4fa1..b27af617ee 100644 --- a/CRM/Import/Form/Summary.php +++ b/CRM/Import/Form/Summary.php @@ -21,7 +21,7 @@ * TODO: CRM-11254 - if preProcess and postProcess functions can be reconciled between the 5 child classes, * those classes can be removed entirely and this class will not need to be abstract */ -abstract class CRM_Import_Form_Summary extends CRM_Core_Form { +abstract class CRM_Import_Form_Summary extends CRM_Import_Forms { /** * Build the form object. diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 05fa85b040..d2b7d98ad8 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -16,6 +16,7 @@ */ use Civi\Api4\UserJob; +use League\Csv\Writer; /** * This class helps the forms within the import flow access submitted & parsed values. @@ -437,15 +438,103 @@ class CRM_Import_Forms extends CRM_Core_Form { * In the future we will use the dataSource object, likely * supporting offset as well. * - * @param int $limit + * @return array|int + * One or more of the statues available - e.g + * CRM_Import_Parser::VALID + * or [CRM_Import_Parser::ERROR, CRM_Import_Parser::CONFLICT] * - * @return array + * @throws \CRM_Core_Exception + * @throws \API_Exception + */ + protected function getDataRows($statuses = [], int $limit = 0): array { + $statuses = (array) $statuses; + return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows(); + } + + /** + * Get the number of rows with the specified status. * + * @param array|int $statuses + * + * @return int + * + * @throws \API_Exception * @throws \CRM_Core_Exception + */ + protected function getRowCount($statuses = []) { + $statuses = (array) $statuses; + return $this->getDataSourceObject()->getRowCount($statuses); + } + + /** + * Outputs and downloads the csv of outcomes from an import job. + * + * This gets the rows from the temp table that match the relevant status + * and output them as a csv. + * * @throws \API_Exception + * @throws \League\Csv\CannotInsertRecord + * @throws \CRM_Core_Exception + */ + public static function outputCSV(): void { + $userJobID = CRM_Utils_Request::retrieveValue('user_job_id', 'Integer', NULL, TRUE); + $status = CRM_Utils_Request::retrieveValue('status', 'String', NULL, TRUE); + $saveFileName = CRM_Import_Parser::saveFileName($status); + + $form = new CRM_Import_Forms(); + $form->controller = new CRM_Core_Controller(); + $form->set('user_job_id', $userJobID); + + $form->getUserJob(); + $writer = Writer::createFromFileObject(new SplTempFileObject()); + $headers = $form->getColumnHeaders(); + if ($headers) { + array_unshift($headers, ts('Reason')); + array_unshift($headers, ts('Line Number')); + $writer->insertOne($headers); + } + $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']); + // Note this might be more inefficient that iterating the result + // set & doing insertOne - possibly something to explore later. + $writer->insertAll($form->getDataRows($status)); + + CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0'); + CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer'); + CRM_Utils_System::setHttpHeader('Content-Type', 'text/csv; charset=UTF-8'); + $writer->output($saveFileName); + CRM_Utils_System::civiExit(); + } + + /** + * When outputting the row as a csv, more the last 2 rows to the start. + * + * This is because the id and status message fields are at the end. It may make sense + * to move them to the start later, when order code cleanup has happened... + * + * @param array $record + */ + public static function reorderOutput(array $record): array { + $rowNumber = array_pop($record); + $message = array_pop($record); + // Also pop off the status - but we are not going to use this at this stage. + array_pop($record); + array_unshift($record, $message); + array_unshift($record, $rowNumber); + return $record; + } + + /** + * Get the url to download the relevant csv file. + * @param string $status + * + * @return string */ - protected function getDataRows(int $limit): array { - return $this->getDataSourceObject()->setLimit($limit)->getRows(); + protected function getDownloadURL(string $status): string { + return CRM_Utils_System::url('civicrm/import/outcome', [ + 'user_job_id' => $this->get('user_job_id'), + 'status' => $status, + 'reset' => 1, + ]); } /** diff --git a/templates/CRM/Contact/Import/Form/Preview.tpl b/templates/CRM/Contact/Import/Form/Preview.tpl index e899d3daf4..5e883e2664 100644 --- a/templates/CRM/Contact/Import/Form/Preview.tpl +++ b/templates/CRM/Contact/Import/Form/Preview.tpl @@ -20,7 +20,7 @@ {if $invalidRowCount}

- {ts 1=$invalidRowCount 2=$downloadErrorRecordsUrl}CiviCRM has detected invalid data or formatting errors in %1 records. If you continue, these records will be skipped. OR, you can download a file with just these problem records - Download Errors. Then correct them in the original import file, cancel this import and begin again at step 1.{/ts} + {ts 1=$invalidRowCount 2=$downloadErrorRecordsUrl|smarty:nodefaults}CiviCRM has detected invalid data or formatting errors in %1 records. If you continue, these records will be skipped. OR, you can download a file with just these problem records - Download Errors. Then correct them in the original import file, cancel this import and begin again at step 1.{/ts}

{/if} @@ -48,7 +48,7 @@ {$invalidRowCount} {ts}Rows with invalid data in one or more fields (for example, invalid email address formatting). These rows will be skipped (not imported).{/ts} {if $invalidRowCount} -
{ts}Download Errors{/ts}
+
{ts}Download Errors{/ts}
{/if} diff --git a/templates/CRM/Contact/Import/Form/Summary.tpl b/templates/CRM/Contact/Import/Form/Summary.tpl index 0b3577e65c..1a4a9baad8 100644 --- a/templates/CRM/Contact/Import/Form/Summary.tpl +++ b/templates/CRM/Contact/Import/Form/Summary.tpl @@ -33,7 +33,7 @@ {ts count=$invalidRowCount plural='CiviCRM has detected invalid data and/or formatting errors in %count records. These records have not been imported.'}CiviCRM has detected invalid data and/or formatting errors in one record. This record has not been imported.{/ts}

- {ts 1=$downloadErrorRecordsUrl}You can Download Errors. You may then correct them, and import the new file with the corrected data.{/ts} + {ts 1=$downloadErrorRecordsUrl|smarty:nodefaults}You can Download Errors. You may then correct them, and import the new file with the corrected data.{/ts}

{/if} @@ -75,7 +75,7 @@ {$invalidRowCount} {ts}Rows with invalid data in one or more fields (for example, invalid email address formatting). These rows will be skipped (not imported).{/ts} {if $invalidRowCount} -
{ts}Download Errors{/ts}
+
{ts}Download Errors{/ts}
{/if} -- 2.25.1