From 5e21b5887ad9c6cc7d66f3ac01972ed7375c9202 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Wed, 11 May 2022 16:58:36 +1200 Subject: [PATCH] [REF] [Import] Move datasource interaction to datasource class --- CRM/Contact/Import/Parser/Contact.php | 73 ++++++------------- CRM/Import/DataSource.php | 46 +++++++++++- CRM/Import/Forms.php | 4 + CRM/Import/Parser.php | 28 +++++++ ...ndividual_valid_with_related_no_email.csv# | 1 + .../individual_valid_with_related_email.csv | 2 + ...individual_valid_with_related_no_email.csv | 2 + .../CRM/Contact/Import/Parser/ContactTest.php | 24 ++++-- .../phpunit/CRM/Import/DataSource/CsvTest.php | 7 ++ 9 files changed, 129 insertions(+), 58 deletions(-) create mode 100644 tests/phpunit/CRM/Contact/Import/Form/data/.~lock.individual_valid_with_related_no_email.csv# create mode 100644 tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_email.csv create mode 100644 tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_no_email.csv diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index f7972d1ed7..9412f4387d 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -811,7 +811,13 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { return CRM_Import_Parser::ERROR; } - // sleep(3); + + if (empty($this->_unparsedStreetAddressContacts)) { + $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '', $contactID); + return CRM_Import_Parser::VALID; + } + + // @todo - record unparsed address as 'imported' but the presence of a message is meaningful? return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID); } @@ -2265,7 +2271,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { * @param array $mapper * @param int $mode * @param int $statusID - * @param int $totalRowCount * * @return mixed * @throws \API_Exception|\CRM_Core_Exception @@ -2273,8 +2278,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { public function run( $mapper = [], $mode = self::MODE_PREVIEW, - $statusID = NULL, - $totalRowCount = NULL + $statusID = NULL ) { // TODO: Make the timeout actually work @@ -2290,7 +2294,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $this->_rowCount = 0; $this->_totalCount = 0; - $this->_tableName = $tableName = $this->getUserJob()['metadata']['DataSource']['table_name']; $this->_primaryKeyName = '_id'; $this->_statusFieldName = '_status'; @@ -2298,10 +2301,10 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $this->progressImport($statusID); $startTimestamp = $currTimestamp = $prevTimestamp = time(); } - // get the contents of the temp. import table - $query = "SELECT * FROM $tableName"; + $dataSource = $this->getDataSourceObject(); + $totalRowCount = $dataSource->getRowCount(['new']); if ($mode == self::MODE_IMPORT) { - $query .= " WHERE _status = 'NEW'"; + $dataSource->setStatuses(['new']); } if ($this->_maxLinesToProcess > 0) { // Note this would only be the case in MapForm mode, where it is set to 100 @@ -2313,20 +2316,13 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { // which is the number of columns a row might have. // However, the mapField class may no longer use activeFieldsCount for contact // to be continued.... - $query .= ' LIMIT ' . $this->_maxLinesToProcess; + $dataSource->setLimit($this->_maxLinesToProcess); } - $result = CRM_Core_DAO::executeQuery($query); - - while ($result->fetch()) { - $values = array_values($result->toArray()); + while ($row = $dataSource->getRow()) { + $values = array_values($row); $this->_rowCount++; - /* trim whitespace around the values */ - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - $this->_totalCount++; if ($mode == self::MODE_MAPFIELD) { @@ -2592,29 +2588,21 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } /** - * Set the import status for the given record. - * - * If this is a sql import then the sql table will be used and the update - * will not happen as the relevant fields don't exist in the table - hence - * the checks that statusField & primary key are set. + * Update the status of the import row to reflect the processing outcome. * * @param int $id * @param string $status * @param string $message + * @param int|null $entityID + * Optional created entity ID + * @param array $relatedEntityIDs + * Optional array e.g ['related_contact' => 4] + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ - public function setImportStatus(int $id, string $status, string $message): void { - if ($this->_statusFieldName && $this->_primaryKeyName) { - CRM_Core_DAO::executeQuery(" - UPDATE $this->_tableName - SET $this->_statusFieldName = %1, - {$this->_statusFieldName}Msg = %2 - WHERE $this->_primaryKeyName = %3 - ", [ - 1 => [$status, 'String'], - 2 => [$message, 'String'], - 3 => [$id, 'Integer'], - ]); - } + public function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, array $relatedEntityIDs = []): void { + $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $relatedEntityIDs); } /** @@ -3096,19 +3084,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { return $params; } - /** - * Is the job complete. - * - * This function transitionally accesses the table from the userJob - * directly - but the function should be moved to the dataSource class. - * - * @throws \API_Exception - */ - public function isComplete() { - $tableName = $this->getUserJob()['metadata']['DataSource']['table_name']; - return (bool) CRM_Core_DAO::singleValueQuery("SELECT count(*) FROM $tableName WHERE _status = 'NEW' LIMIT 1"); - } - /** * Validate the import values. * diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index 1ef9a69c5e..5d72499ae1 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -442,12 +442,55 @@ abstract class CRM_Import_DataSource { protected function addTrackingFieldsToTable(string $tableName): void { CRM_Core_DAO::executeQuery(" ALTER TABLE $tableName + ADD COLUMN _entity_id INT, + ADD COLUMN _related_entity_ids JSON, ADD COLUMN _status VARCHAR(32) DEFAULT 'NEW' NOT NULL, - ADD COLUMN _statusMsg TEXT, + ADD COLUMN _status_message TEXT, ADD COLUMN _id INT PRIMARY KEY NOT NULL AUTO_INCREMENT" ); } + /** + * Has the import job completed. + * + * @return bool + * True if no rows remain to be imported. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + public function isCompleted(): bool { + return (bool) $this->getRowCount(['new']); + } + + /** + * Update the status of the import row to reflect the processing outcome. + * + * @param int $id + * @param string $status + * @param string $message + * @param int|null $entityID + * Optional created entity ID + * @param array $relatedEntityIDs + * Optional array e.g ['related_contact' => 4] + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + public function updateStatus(int $id, string $status, string $message, ? int $entityID = NULL, array $relatedEntityIDs = []): void { + $sql = 'UPDATE ' . $this->getTableName() . ' SET _status = %1, _status_message = %2 '; + $params = [1 => [$status, 'String'], 2 => [$message, 'String']]; + if ($entityID) { + $sql .= ', _entity_id = %3'; + $params[3] = [$entityID, 'Integer']; + } + if ($relatedEntityIDs) { + $sql .= ', _related_entities = %4'; + $params[4] = [json_encode($relatedEntityIDs), 'String']; + } + CRM_Core_DAO::executeQuery($sql . ' WHERE _id = ' . $id, $params); + } + /** * * @throws \API_Exception @@ -473,6 +516,7 @@ abstract class CRM_Import_DataSource { CRM_Import_Parser::DUPLICATE => ['duplicate'], CRM_Import_Parser::NO_MATCH => ['invalid_no_match'], CRM_Import_Parser::UNPARSED_ADDRESS_WARNING => ['warning_unparsed_address'], + 'new' => ['new'], ]; } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 3816383234..76d9868a35 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -518,6 +518,10 @@ class CRM_Import_Forms extends CRM_Core_Form { $message = array_pop($record); // Also pop off the status - but we are not going to use this at this stage. array_pop($record); + // Related entities + array_pop($record); + // Entity_id + array_pop($record); array_unshift($record, $message); array_unshift($record, $rowNumber); return $record; diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 1880d33b9a..739cd6552e 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -85,6 +85,22 @@ abstract class CRM_Import_Parser { ->first(); } + /** + * Get the relevant datasource object. + * + * @return \CRM_Import_DataSource|null + * + * @throws \API_Exception + */ + protected function getDataSourceObject(): ?CRM_Import_DataSource { + $className = $this->getSubmittedValue('dataSource'); + if ($className) { + /* @var CRM_Import_DataSource $dataSource */ + return new $className($this->getUserJobID()); + } + return NULL; + } + /** * Get the submitted value, as stored on the user job. * @@ -98,6 +114,18 @@ abstract class CRM_Import_Parser { return $this->getUserJob()['metadata']['submitted_values'][$fieldName]; } + /** + * Has the import completed. + * + * @return bool + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + public function isComplete() :bool { + return $this->getDataSourceObject()->isCompleted(); + } + /** * Get configured contact type. * diff --git a/tests/phpunit/CRM/Contact/Import/Form/data/.~lock.individual_valid_with_related_no_email.csv# b/tests/phpunit/CRM/Contact/Import/Form/data/.~lock.individual_valid_with_related_no_email.csv# new file mode 100644 index 0000000000..8d0a2114c2 --- /dev/null +++ b/tests/phpunit/CRM/Contact/Import/Form/data/.~lock.individual_valid_with_related_no_email.csv# @@ -0,0 +1 @@ +,eileen,eileen-laptop,11.05.2022 11:24,file:///home/eileen/.config/libreoffice/4; \ No newline at end of file diff --git a/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_email.csv b/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_email.csv new file mode 100644 index 0000000000..3e711e3731 --- /dev/null +++ b/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_email.csv @@ -0,0 +1,2 @@ +Player First Name,Player Last name,Mother – email +Susie,Jones,mum@example.org diff --git a/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_no_email.csv b/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_no_email.csv new file mode 100644 index 0000000000..14d810f361 --- /dev/null +++ b/tests/phpunit/CRM/Contact/Import/Form/data/individual_valid_with_related_no_email.csv @@ -0,0 +1,2 @@ +Player First Name,Player Last name,Mother – phone +Blake,Jackson,911 diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php index 268a8b046f..a63ed3bde4 100644 --- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php +++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php @@ -473,9 +473,8 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { $processor = new CRM_Import_ImportProcessor(); $processor->setMappingFields($mapping); - $processor->setUserJobID($this->getUserJobID([ - 'mapper' => $mapperInput, - ])); + $userJobID = $this->getUserJobID(['mapper' => $mapperInput]); + $processor->setUserJobID($userJobID); $importer = $processor->getImporterObject(); $contactValues = [ @@ -1020,10 +1019,9 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { foreach ($fields as $index => $field) { $mapper[] = [$field, $mapperLocType[$index] ?? NULL, $field === 'phone' ? 1 : NULL]; } + $userJobID = $this->getUserJobID(['mapper' => $mapper]); $parser = new CRM_Contact_Import_Parser_Contact($fields, $mapperLocType); - $parser->setUserJobID($this->getUserJobID([ - 'mapper' => $mapper, - ])); + $parser->setUserJobID($userJobID); $parser->_dedupeRuleGroupID = $ruleGroupId; $parser->_onDuplicate = $onDuplicateAction; $parser->init(); @@ -1157,17 +1155,27 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { * @throws \Civi\API\Exception\UnauthorizedException */ protected function getUserJobID($submittedValues = []) { - return UserJob::create()->setValues([ + $userJobID = UserJob::create()->setValues([ 'metadata' => [ 'submitted_values' => array_merge([ 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, 'contactSubType' => '', 'doGeocodeAddress' => 0, + 'dataSource' => 'CRM_Import_DataSource_SQL', + 'sqlQuery' => 'SELECT first_name FROM civicrm_contact', ], $submittedValues), ], 'status_id:name' => 'draft', 'type_id:name' => 'contact_import', ])->execute()->first()['id']; + if ($submittedValues['dataSource'] ?? NULL === 'CRM_Import_DataSource') { + $dataSource = new CRM_Import_DataSource_CSV($userJobID); + } + else { + $dataSource = new CRM_Import_DataSource_SQL($userJobID); + } + $dataSource->initialize(); + return $userJobID; } /** @@ -1186,9 +1194,9 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { 'skipColumnHeader' => TRUE, 'fieldSeparator' => ',', 'mapper' => $mapper, + 'dataSource' => 'CRM_Import_DataSource_CSV', ]); $dataSource = new CRM_Import_DataSource_CSV($userJobID); - $dataSource->initialize(); $parser = new CRM_Contact_Import_Parser_Contact(); $parser->setUserJobID($userJobID); $parser->init(); diff --git a/tests/phpunit/CRM/Import/DataSource/CsvTest.php b/tests/phpunit/CRM/Import/DataSource/CsvTest.php index 594060c0ed..aedd68dc40 100644 --- a/tests/phpunit/CRM/Import/DataSource/CsvTest.php +++ b/tests/phpunit/CRM/Import/DataSource/CsvTest.php @@ -130,6 +130,9 @@ class CRM_Import_DataSource_CsvTest extends CiviUnitTestCase { * edge case. Note if it has more than one column then the blank line gets * skipped because of some checking for column-count matches in the import, * and so you don't hit the current fail. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ public function testBlankLineAtEnd(): void { $form = $this->submitDatasourceForm('blankLineAtEnd.csv'); @@ -146,6 +149,9 @@ class CRM_Import_DataSource_CsvTest extends CiviUnitTestCase { * @param string $csvFileName * * @return \CRM_Contact_Import_Form_DataSource + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ protected function submitDatasourceForm(string $csvFileName): CRM_Contact_Import_Form_DataSource { $_GET['dataSource'] = 'CRM_Import_DataSource_CSV'; @@ -156,6 +162,7 @@ class CRM_Import_DataSource_CsvTest extends CiviUnitTestCase { ], 'skipColumnHeader' => TRUE, 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'dataSource' => 'CRM_Import_DataSource_CSV', ]); $form->buildForm(); $form->postProcess(); -- 2.25.1