From 2a4de39fe206188b85646f3a80e8401947d02e23 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Sun, 5 Jun 2022 12:43:41 +1200 Subject: [PATCH] Add import-specific tracking fields --- CRM/Contact/Import/Parser/Contact.php | 59 +++++++++++-------- CRM/Import/DataSource.php | 53 ++++++++++++++--- CRM/Import/Parser.php | 16 ++++- .../CRM/Contact/Import/Parser/ContactTest.php | 30 +++++++--- 4 files changed, 116 insertions(+), 42 deletions(-) diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 05448fb3bc..2c4d88ef17 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -85,6 +85,18 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } } + /** + * Get the fields to track the import. + * + * @return array + */ + public function getTrackingFields(): array { + return [ + 'related_contact_created' => 'INT COMMENT "Number of related contacts created"', + 'related_contact_matched' => 'INT COMMENT "Number of related contacts found (& potentially updated)"', + ]; + } + /** * Is street address parsing enabled for the site. */ @@ -172,7 +184,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { * @return bool * the result of this processing * - * @throws \CiviCRM_API3_Exception * @throws \CRM_Core_Exception * @throws \API_Exception */ @@ -194,23 +205,17 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } } - $params['contact_sub_type'] = $this->getContactSubType() ?: ($params['contact_sub_type'] ?? NULL); - [$formatted, $params] = $this->processContact($params, $formatted, TRUE); - // Get contact id to format common data in update/fill mode, - // prioritising a dedupe rule check over an external_identifier check, but falling back on ext id. - //format common data, CRM-4062 $this->formatCommonData($params, $formatted); $newContact = $this->createContact($formatted, $params['id'] ?? NULL); - $this->createdContacts[$newContact->id] = $contactID = $newContact->id; + $contactID = $newContact->id; if ($contactID) { // call import hook $currentImportID = end($values); - $hookParams = [ 'contactID' => $contactID, 'importID' => $currentImportID, @@ -218,7 +223,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { 'fieldHeaders' => $this->_mapperKeys, 'fields' => $this->_activeFields, ]; - CRM_Utils_Hook::import('Contact', 'process', $this, $hookParams); } @@ -234,17 +238,35 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { if (empty($formatting['id']) || $this->isUpdateExistingContacts()) { $relatedNewContact = $this->createContact($formatting, $formatting['id']); - $relContactId = $relatedNewContact->id; - $this->createdContacts[$relContactId] = $relContactId; + $formatting['id'] = $relatedNewContact->id; + } + if (empty($relatedContacts[$formatting['id']])) { + $relatedContacts[$formatting['id']] = 'new'; } - $this->createRelationship($key, $relContactId, $primaryContactId); + + $this->createRelationship($key, $formatting['id'], $primaryContactId); } } catch (CRM_Core_Exception $e) { $this->setImportStatus($rowNumber, $this->getStatus($e->getErrorCode()), $e->getMessage()); return FALSE; } - $this->setImportStatus($rowNumber, $this->getStatus(CRM_Import_Parser::VALID), $this->getSuccessMessage(), $contactID); + // We can probably stop catching this once https://github.com/civicrm/civicrm-core/pull/23471 + // is merged - testImportParserWithExternalIdForRelationship will confirm.... + catch (CiviCRM_API3_Exception $e) { + $this->setImportStatus($rowNumber, $this->getStatus($e->getErrorCode()), $e->getMessage()); + return FALSE; + } + $extraFields = ['related_contact_created' => 0, 'related_contact_matched' => 0]; + foreach ($relatedContacts as $key => $outcome) { + if ($outcome === 'new') { + $extraFields['related_contact_created']++; + } + else { + $extraFields['related_contact_matched']++; + } + } + $this->setImportStatus($rowNumber, $this->getStatus(CRM_Import_Parser::VALID), $this->getSuccessMessage(), $contactID, $extraFields); return CRM_Import_Parser::VALID; } @@ -499,15 +521,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { } } - /** - * Get the array of successfully imported contact id's - * - * @return array - */ - public function getImportedContacts() { - return $this->createdContacts; - } - /** * Build error-message containing error-fields * @@ -1878,7 +1891,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { */ protected function processContact(array $params, array $formatted, bool $isMainContact): array { $params['id'] = $formatted['id'] = $this->lookupContactID($params, $isMainContact); - if ($params['id'] && $params['contact_sub_type']) { + if ($params['id'] && !empty($params['contact_sub_type'])) { $contactSubType = Contact::get(FALSE) ->addWhere('id', '=', $params['id']) ->addSelect('contact_sub_type') diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index 9175936548..47bc3f6e1e 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -435,21 +435,56 @@ abstract class CRM_Import_DataSource { * could be cases where it still clashes but time didn't tell in this case) * 2) the show fields query used to get the column names excluded the * administrative fields, relying on this convention. - * 3) we have the capitalisation on _statusMsg - @todo change to _status_message + * 3) we have the capitalisation on _statusMsg - @param string $tableName * - * @param string $tableName + * @throws \API_Exception + * @todo change to _status_message */ protected function addTrackingFieldsToTable(string $tableName): void { CRM_Core_DAO::executeQuery(" ALTER TABLE $tableName ADD COLUMN _entity_id INT, - ADD COLUMN _related_entity_ids LONGTEXT, + " . $this->getAdditionalTrackingFields() . " ADD COLUMN _status VARCHAR(32) DEFAULT 'NEW' NOT NULL, ADD COLUMN _status_message TEXT, ADD COLUMN _id INT PRIMARY KEY NOT NULL AUTO_INCREMENT" ); } + /** + * Get any additional import specific tracking fields. + * + * @throws \API_Exception + */ + private function getAdditionalTrackingFields(): string { + $sql = ''; + $fields = $this->getParser()->getTrackingFields(); + foreach ($fields as $fieldName => $spec) { + $sql .= 'ADD COLUMN _' . $fieldName . ' ' . $spec . ','; + } + return $sql; + } + + /** + * Get the import parser. + * + * @return CRM_Import_Parser + * + * @throws \API_Exception + */ + private function getParser() { + $parserClass = ''; + foreach (CRM_Core_BAO_UserJob::getTypes() as $type) { + if ($this->getUserJob()['type_id'] === $type['id']) { + $parserClass = $type['class']; + } + } + /* @var \CRM_Import_Parser */ + $parser = new $parserClass(); + $parser->setUserJobID($this->getUserJobID()); + return $parser; + } + /** * Has the import job completed. * @@ -471,22 +506,24 @@ abstract class CRM_Import_DataSource { * @param string $message * @param int|null $entityID * Optional created entity ID - * @param array $relatedEntityIDs + * @param array $additionalFields * Optional array e.g ['related_contact' => 4] * * @throws \API_Exception * @throws \CRM_Core_Exception */ - public function updateStatus(int $id, string $status, string $message, ? int $entityID = NULL, array $relatedEntityIDs = []): void { + public function updateStatus(int $id, string $status, string $message, ? int $entityID = NULL, array $additionalFields = []): void { $sql = 'UPDATE ' . $this->getTableName() . ' SET _status = %1, _status_message = %2 '; $params = [1 => [$status, 'String'], 2 => [$message, 'String']]; if ($entityID) { $sql .= ', _entity_id = %3'; $params[3] = [$entityID, 'Integer']; } - if ($relatedEntityIDs) { - $sql .= ', _related_entities = %4'; - $params[4] = [json_encode($relatedEntityIDs), 'String']; + $nextParam = 4; + foreach ($additionalFields as $fieldName => $value) { + $sql .= ', _' . $fieldName . ' = %' . $nextParam; + $params[$nextParam] = is_numeric($value) ? [$value, 'Int'] : [json_encode($value), 'String']; + $nextParam++; } CRM_Core_DAO::executeQuery($sql . ' WHERE _id = ' . $id, $params); } diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index d02d9c1f47..6aa63f1094 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -109,6 +109,14 @@ abstract class CRM_Import_Parser { */ private $availableCountries; + /** + * + * @return array + */ + public function getTrackingFields(): array { + return []; + } + /** * Get User Job. * @@ -1842,12 +1850,16 @@ abstract class CRM_Import_Parser { * @param string $message * @param int|null $entityID * Optional created entity ID + * @param array $additionalFields + * Additional fields to be tracked * * @noinspection PhpDocMissingThrowsInspection * @noinspection PhpUnhandledExceptionInspection + * @throws \API_Exception + * @throws \CRM_Core_Exception */ - protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL): void { - $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID); + protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, $additionalFields = []): void { + $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields); } /** diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php index a7beafebf0..ac3216dcbb 100644 --- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php +++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php @@ -49,6 +49,13 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { */ private $relationships = []; + /** + * User Job ID. + * + * @var int + */ + private $userJobID; + /** * Tear down after test. */ @@ -1125,7 +1132,6 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { */ public function testImportCountryStateCounty(): void { $childKey = $this->getRelationships()['Child of']['id'] . '_a_b'; - // @todo - rows that don't work yet are set to do_not_import. $addressCustomGroupID = $this->createCustomGroup(['extends' => 'Address', 'name' => 'Address']); $contactCustomGroupID = $this->createCustomGroup(['extends' => 'Contact', 'name' => 'Contact']); $addressCustomFieldID = $this->createCountryCustomField(['custom_group_id' => $addressCustomGroupID])['id']; @@ -1175,6 +1181,11 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { $this->assertEquals(1640, $contact['address'][0]['state_province_id']); } $this->assertCount(2, $contacts); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $dataSource->setOffset(4); + $dataSource->setLimit(1); + $row = $dataSource->getRow(); + $this->assertEquals(1, $row['_related_contact_matched']); } /** @@ -1385,7 +1396,7 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { * * @throw \Exception */ - public function testImportFill() { + public function testImportFill(): void { // Create a custom field group for testing. $this->createCustomGroup([ 'title' => 'importFillGroup', @@ -1623,13 +1634,14 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { ]; } } - $userJobID = $this->getUserJobID(['mapper' => $mapper, 'onDuplicate' => $onDuplicateAction, 'dedupe_rule_id' => $ruleGroupId]); - $parser = new CRM_Contact_Import_Parser_Contact($fields); - $parser->setUserJobID($userJobID); + $this->userJobID = $this->getUserJobID(['mapper' => $mapper, 'onDuplicate' => $onDuplicateAction, 'dedupe_rule_id' => $ruleGroupId]); + $parser = new CRM_Contact_Import_Parser_Contact(); + $parser->setUserJobID($this->userJobID); $parser->_dedupeRuleGroupID = $ruleGroupId; $parser->init(); + $result = $parser->import($values); - $dataSource = new CRM_Import_DataSource_CSV($userJobID); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); if ($result === FALSE && $expectedResult !== FALSE) { // Import is moving away from returning a status - this is a better way to check $this->assertGreaterThan(0, $dataSource->getRowCount([$expectedResult])); @@ -2035,15 +2047,15 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { $form = $this->getFormObject('CRM_Contact_Import_Form_DataSource', $submittedValues); $form->buildForm(); $form->postProcess(); - $userJobID = $form->getUserJobID(); + $this->userJobID = $form->getUserJobID(); /* @var CRM_Contact_Import_Form_MapField $form */ $form = $this->getFormObject('CRM_Contact_Import_Form_MapField', $submittedValues); - $form->setUserJobID($userJobID); + $form->setUserJobID($this->userJobID); $form->buildForm(); $form->postProcess(); /* @var CRM_Contact_Import_Form_MapField $form */ $form = $this->getFormObject('CRM_Contact_Import_Form_Preview', $submittedValues); - $form->setUserJobID($userJobID); + $form->setUserJobID($this->userJobID); $form->buildForm(); try { -- 2.25.1