From 2b8037052e5724858c864084123c4af0c0020f0a Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Thu, 16 Mar 2023 17:48:46 +1300 Subject: [PATCH] Add url to UserJob data & declare import Type on MapField screens The DataSource screen currently declares the type so this makes it available on MapField Minor tidy up/ standardisation on Contact defaultValues This also allows the parent to override the values, e.g if loaded from a template Fix BAO to support Template UserJobs - do not save non-template fields - delete templates on Mapping deletes Add support for UserJob templates at the form level Fix for strict typing --- CRM/Contact/Import/Form/DataSource.php | 57 +---- CRM/Contact/Import/Form/Preview.php | 9 + CRM/Contact/Import/Form/Summary.php | 7 + CRM/Core/BAO/UserJob.php | 57 ++++- CRM/Custom/Import/Form/DataSource.php | 16 +- CRM/Import/DataSource.php | 2 +- CRM/Import/Form/DataSource.php | 66 +++++- CRM/Import/Form/MapField.php | 35 +-- CRM/Import/Forms.php | 214 +++++++++++++++++- CRM/Report/Form/Contact/Summary.php | 2 +- ext/civiimport/civiimport.php | 5 +- templates/CRM/Contact/Import/Form/Summary.tpl | 8 +- templates/CRM/Import/Form/DataSource.tpl | 2 +- .../CRM/Contact/Import/Form/MapFieldTest.php | 2 +- .../CRM/Import/DataSource/FormsTest.php | 181 +++++++++++++++ tests/phpunit/CiviTest/CiviUnitTestCase.php | 18 +- 16 files changed, 569 insertions(+), 112 deletions(-) create mode 100644 tests/phpunit/CRM/Import/DataSource/FormsTest.php diff --git a/CRM/Contact/Import/Form/DataSource.php b/CRM/Contact/Import/Form/DataSource.php index 84530dc29f..f8755c029e 100644 --- a/CRM/Contact/Import/Form/DataSource.php +++ b/CRM/Contact/Import/Form/DataSource.php @@ -80,58 +80,11 @@ class CRM_Contact_Import_Form_DataSource extends CRM_Import_Form_DataSource { * @return array * reference to the array of default values */ - public function setDefaultValues() { - $defaults = parent::setDefaultValues(); - $defaults['contactType'] = 'Individual'; - $defaults['disableUSPS'] = TRUE; - - if ($this->get('loadedMapping')) { - $defaults['savedMapping'] = $this->get('loadedMapping'); - } - - return $defaults; - } - - /** - * Call the DataSource's postProcess method. - * - * @throws \CRM_Core_Exception - */ - public function postProcess() { - $this->controller->resetPage('MapField'); - $this->processDatasource(); - // @todo - this params are being set here because they were / possibly still - // are in some places being accessed by forms later in the flow - // ie CRM_Contact_Import_Form_MapField, CRM_Contact_Import_Form_Preview - // which was the old way of saving values submitted on this form such that - // the other forms could access them. Now they should use - // `getSubmittedValue` or simply not get them if the only - // reason is to pass to the Parser which can itself - // call 'getSubmittedValue' - // Once the mentioned forms no longer call $this->get() all this 'setting' - // is obsolete. - $storeParams = [ - 'savedMapping' => $this->getSubmittedValue('savedMapping'), - ]; - - foreach ($storeParams as $storeName => $value) { - $this->set($storeName, $value); - } - } - - /** - * General function for handling invalid configuration. - * - * I was going to statusBounce them all but when I tested I was 'bouncing' to weird places - * whereas throwing an exception gave no behaviour change. So, I decided to centralise - * and we can 'flip the switch' later. - * - * @param $message - * - * @throws \CRM_Core_Exception - */ - protected function invalidConfig($message) { - throw new CRM_Core_Exception($message); + public function setDefaultValues(): array { + return array_merge([ + 'contactType' => 'Individual', + 'disableUSPS' => TRUE, + ], parent::setDefaultValues()); } /** diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index cfc55e5c03..2789484483 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -23,6 +23,15 @@ use Civi\Api4\Tag; */ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { + /** + * Get the name of the type to be stored in civicrm_user_job.type_id. + * + * @return string + */ + public function getUserJobType(): string { + return 'contact_import'; + } + /** * Build the form object. */ diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php index 2b606fc9de..15460d4be0 100644 --- a/CRM/Contact/Import/Form/Summary.php +++ b/CRM/Contact/Import/Form/Summary.php @@ -33,6 +33,13 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Forms { $this->setTitle($userJob['job_type:label']); $onDuplicate = $userJob['metadata']['submitted_values']['onDuplicate']; $this->assign('dupeError', FALSE); + $importBaseURL = $this->getUserJobInfo()['url'] ?? NULL; + $this->assign('templateURL', ($importBaseURL && $this->getTemplateID()) ? CRM_Utils_System::url($importBaseURL, ['template_id' => $this->getTemplateID(), 'reset' => 1]) : ''); + // This can be overridden by Civi-Import so that the Download url + // links that go to SearchKit open in a new tab. + $this->assign('isOpenResultsInNewTab'); + $this->assign('allRowsUrl'); + $this->assign('importedRowsUrl'); if ($onDuplicate === CRM_Import_Parser::DUPLICATE_UPDATE) { $this->assign('dupeActionString', ts('These records have been updated with the imported data.')); diff --git a/CRM/Core/BAO/UserJob.php b/CRM/Core/BAO/UserJob.php index 63fc46c0d5..9979e6c73e 100644 --- a/CRM/Core/BAO/UserJob.php +++ b/CRM/Core/BAO/UserJob.php @@ -15,14 +15,17 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\Mapping; use Civi\Api4\UserJob; use Civi\Core\ClassScanner; +use Civi\Core\Event\PreEvent; +use Civi\Core\HookInterface; use Civi\UserJob\UserJobInterface; /** * This class contains user jobs functionality. */ -class CRM_Core_BAO_UserJob extends CRM_Core_DAO_UserJob implements \Civi\Core\HookInterface { +class CRM_Core_BAO_UserJob extends CRM_Core_DAO_UserJob implements HookInterface { /** * Check on the status of a queue. @@ -63,6 +66,58 @@ class CRM_Core_BAO_UserJob extends CRM_Core_DAO_UserJob implements \Civi\Core\Ho } } + /** + * Enforce template expectations by unsetting non-template variables. + * + * Also delete the template if the Mapping is deleted. + * + * @param \Civi\Core\Event\PreEvent $event + * + * @noinspection PhpUnused + * @throws \CRM_Core_Exception + */ + public static function on_hook_civicrm_pre(PreEvent $event): void { + if ($event->entity === 'UserJob' && + (!empty($event->params['is_template']) + || ($event->action === 'edit' && self::isTemplate($event->params['id'])) + )) { + $params = &$event->params; + if (empty($params['name']) && empty($params['id'])) { + throw new CRM_Core_Exception('Name is required for template user job'); + } + if ($params['metadata']['submitted_values']['dataSource'] ?? NULL === 'CRM_Import_DataSource_SQL') { + // This contains path information that we are better to ditch at this point. + // Ideally we wouldn't save this in submitted values - but just use it. + unset($params['metadata']['submitted_values']['uploadFile']); + } + // This contains information about the import-specific data table. + unset($params['metadata']['DataSource']['table_name']); + // Do not keep values about updating the Mapping/UserJob template. + unset($params['metadata']['MapField']['saveMapping'], $params['metadata']['MapField']['updateMapping']); + } + + // If the related mapping is deleted then delete the UserJob template + // This almost never happens in practice... + if ($event->entity === 'Mapping' && $event->action === 'delete') { + $mappingName = Mapping::get(FALSE)->addWhere('id', '=', $event->id)->addSelect('name')->execute()->first()['name']; + UserJob::delete(FALSE)->addWhere('name', '=', 'import_' . $mappingName)->execute(); + } + } + + /** + * Is this id a Template. + * + * @param int $id + * + * @return bool + * @throws \CRM_Core_Exception + */ + private static function isTemplate(int $id) : bool { + return (bool) UserJob::get(FALSE)->addWhere('id', '=', $id) + ->addWhere('is_template', '=', 1) + ->selectRowCount()->execute()->rowCount; + } + private static function findUserJobId(string $queueName): ?int { if (CRM_Core_Config::isUpgradeMode()) { return NULL; diff --git a/CRM/Custom/Import/Form/DataSource.php b/CRM/Custom/Import/Form/DataSource.php index 90f5174e42..b50b382f20 100644 --- a/CRM/Custom/Import/Form/DataSource.php +++ b/CRM/Custom/Import/Form/DataSource.php @@ -95,17 +95,11 @@ class CRM_Custom_Import_Form_DataSource extends CRM_Import_Form_DataSource { * @throws \CRM_Core_Exception */ public function setDefaultValues(): array { - parent::setDefaultValues(); - $defaults['contactType'] = 'Individual'; - // Perhaps never used, but permits url passing of the group. - $defaults['multipleCustomData'] = CRM_Utils_Request::retrieve('id', 'Positive', $this); - - $loadedMapping = $this->get('loadedMapping'); - if ($loadedMapping) { - $defaults['savedMapping'] = $loadedMapping; - } - - return $defaults; + return array_merge(parent::setDefaultValues(), [ + 'contactType' => 'Individual', + // Perhaps never used, but permits url passing of the group. + 'multipleCustomData' => CRM_Utils_Request::retrieve('id', 'Positive', $this), + ]); } /** diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index b6b1f00777..b44933274a 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -375,7 +375,7 @@ abstract class CRM_Import_DataSource { */ protected function getTableName(): ?string { // The old name is still stored... - $tableName = $this->getDataSourceMetadata()['table_name']; + $tableName = $this->getDataSourceMetadata()['table_name'] ?? NULL; if (!$tableName) { return NULL; } diff --git a/CRM/Import/Form/DataSource.php b/CRM/Import/Form/DataSource.php index 87f9b933ce..3668ac1dae 100644 --- a/CRM/Import/Form/DataSource.php +++ b/CRM/Import/Form/DataSource.php @@ -14,13 +14,24 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\Mapping; use Civi\Api4\Utils\CoreUtil; +use Civi\Api4\UserJob; /** * Base class for upload-only import forms (all but Contact import). */ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { + /** + * Values loaded from a saved UserJob template. + * + * Within Civi-Import it is possible to save a UserJob with is_template = 1. + * + * @var array + */ + protected $templateValues = []; + /** * Set variables up before form is built. */ @@ -43,6 +54,16 @@ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { return (string) CoreUtil::getInfoItem($this->getBaseEntity(), 'title'); } + /** + * Get the mapping ID that is being loaded. + * + * @return int|null + * @throws \CRM_Core_Exception + */ + public function getSavedMappingID(): ?int { + return $this->getSubmittedValue('savedMapping') ?: NULL; + } + /** * Get the import entity plural (translated). * @@ -71,16 +92,19 @@ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { CRM.$('#data-source-form-block').toggle()", ]); } + if ($this->getTemplateID()) { + $this->setTemplateDefaults(); + } $this->add('select', 'dataSource', ts('Data Source'), $this->getDataSources(), TRUE, ['onchange' => 'buildDataSourceFormBlock(this.value);'] ); $mappingArray = CRM_Core_BAO_Mapping::getCreateMappingValues('Import ' . $this->getBaseEntity()); - $this->add('select', 'savedMapping', ts('Saved Field Mapping'), ['' => ts('- select -')] + $mappingArray); - if ($loadedMapping = $this->get('loadedMapping')) { - $this->setDefaults(['savedMapping' => $loadedMapping]); + $savedMappingElement = $this->add('select', 'savedMapping', ts('Saved Field Mapping'), ['' => ts('- select -')] + $mappingArray); + if ($this->getTemplateID()) { + $savedMappingElement->freeze(); } //build date formats @@ -113,8 +137,7 @@ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { return array_merge($this->dataSourceDefaults, [ 'dataSource' => $this->getDefaultDataSource(), 'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP, - ]); - + ], $this->templateValues); } /** @@ -153,7 +176,7 @@ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { * * @param array $names */ - protected function storeFormValues($names) { + protected function storeFormValues(array $names): void { foreach ($names as $name) { $this->set($name, $this->controller->exportValue($this->_name, $name)); } @@ -178,9 +201,36 @@ abstract class CRM_Import_Form_DataSource extends CRM_Import_Forms { } /** - * Process the datasource submission - setting up the job and data source. + * Load default values from the relevant template if one is passed in via the url. * - * @throws \CRM_Core_Exception + * We need to create and UserJob at this point as the relevant values + * go beyond the first DataSource screen. + * + * @return array + * @noinspection PhpUnhandledExceptionInspection + * @noinspection PhpDocMissingThrowsInspection + */ + public function setTemplateDefaults(): array { + $templateID = $this->getTemplateID(); + if ($templateID && !$this->getUserJobID()) { + $userJob = UserJob::get(FALSE)->addWhere('id', '=', $templateID)->execute()->first(); + $userJobName = $userJob['name']; + // Strip off import_ prefix from UserJob.name + $mappingName = substr($userJobName, 7); + $mappingID = Mapping::get(FALSE)->addWhere('name', '=', $mappingName)->addSelect('id')->execute()->first()['id']; + // Unset fields that should not be copied over. + unset($userJob['id'], $userJob['name'], $userJob['created_id'], $userJob['created_date'], $userJob['expires_date'], $userJob['is_template'], $userJob['queue_id'], $userJob['start_date'], $userJob['end_date']); + $userJob['metadata']['template_id'] = $templateID; + $userJobID = UserJob::create(FALSE)->setValues($userJob)->execute()->first()['id']; + $this->set('user_job_id', $userJobID); + $userJob['metadata']['submitted_values']['savedMapping'] = $mappingID; + $this->templateValues = $userJob['metadata']['submitted_values']; + } + return []; + } + + /** + * Process the datasource submission - setting up the job and data source. */ protected function processDatasource(): void { try { diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index e888ee0af0..7c7b431d41 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -80,8 +80,11 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { * @noinspection PhpUnhandledExceptionInspection */ public function postProcess() { - $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); + // This savedMappingID is the one selected on DataSource. It will be overwritten in saveMapping if any + // action was taken on it. + $this->savedMappingID = $this->getSubmittedValue('savedMapping') ?: NULL; $this->saveMapping(); + $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); $parser = $this->getParser(); $parser->init(); $parser->validate(); @@ -154,9 +157,6 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { // @todo we should stop doing this - the passed in value should be fine, confirmed OK in contact import. $savedMapping = $this->get('savedMapping'); $mappingName = (string) civicrm_api3('Mapping', 'getvalue', ['id' => $savedMappingID, 'return' => 'name']); - // @todo - this should go too - used when going back to the DataSource form but it should - // access the job. - $this->set('loadedMapping', $savedMapping); $this->add('hidden', 'mappingId', $savedMapping); $this->addElement('checkbox', 'updateMapping', ts('Update this field mapping'), NULL); @@ -252,10 +252,12 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { protected function saveMapping(): void { //Updating Mapping Records if ($this->getSubmittedValue('updateMapping')) { + $savedMappingID = (int) $this->getSubmittedValue('mappingId'); foreach (array_keys($this->getColumnHeaders()) as $i) { - $this->saveMappingField((int) $this->getSubmittedValue('mappingId'), $i, TRUE); + $this->saveMappingField($savedMappingID, $i, TRUE); } - $this->updateUserJobMetadata('mapping', ['id' => (int) $this->getSubmittedValue('mappingId')]); + $this->setSavedMappingID($savedMappingID); + $this->updateUserJobMetadata('Template', ['mapping_id' => (int) $this->getSubmittedValue('mappingId')]); } //Saving Mapping Details and Records if ($this->getSubmittedValue('saveMapping')) { @@ -264,7 +266,8 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { 'description' => $this->getSubmittedValue('saveMappingDesc'), 'mapping_type_id:name' => $this->getMappingTypeName(), ])->execute()->first()['id']; - $this->updateUserJobMetadata('MapField', ['mapping_id' => $savedMappingID]); + $this->setSavedMappingID($savedMappingID); + $this->updateUserJobMetadata('Template', ['mapping_id' => $savedMappingID]); foreach (array_keys($this->getColumnHeaders()) as $i) { $this->saveMappingField($savedMappingID, $i, FALSE); } @@ -379,20 +382,14 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { $this->add('text', 'saveMappingDesc', ts('Description')); } else { - // @todo we should stop doing this - the passed in value should be fine, confirmed OK in contact import. - $savedMapping = $this->get('savedMapping'); - $mappingName = (string) civicrm_api3('Mapping', 'getvalue', ['id' => $savedMappingID, 'return' => 'name']); - // @todo - this should go too - used when going back to the DataSource form but it should - // access the job. - $this->set('loadedMapping', $savedMapping); - $this->add('hidden', 'mappingId', $savedMapping); + $this->add('hidden', 'mappingId', $savedMappingID); $this->addElement('checkbox', 'updateMapping', ts('Update this field mapping'), NULL); $saveDetailsName = ts('Save as a new field mapping'); $this->add('text', 'saveMappingName', ts('Name')); $this->add('text', 'saveMappingDesc', ts('Description')); } - $this->assign('savedMappingName', $mappingName ?? NULL); + $this->assign('savedMappingName', $this->getMappingName()); $this->addElement('checkbox', 'saveMapping', $saveDetailsName, NULL); $this->addFormRule(['CRM_Import_Form_MapField', 'mappingRule']); } @@ -496,12 +493,4 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { return $this->defaultFromHeader($columnHeader, $headerPatterns); } - /** - * @return int - */ - protected function getSavedMappingID(): int { - $savedMappingID = (int) ($this->getUserJob()['metadata']['MapField']['mapping_id'] ?? $this->getSubmittedValue('savedMapping')); - return $savedMappingID; - } - } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 10df71e5b7..31df4826e3 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -15,6 +15,7 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\Mapping; use Civi\Api4\UserJob; use League\Csv\Writer; @@ -23,6 +24,12 @@ use League\Csv\Writer; */ class CRM_Import_Forms extends CRM_Core_Form { + + /** + * @var int + */ + protected $templateID; + /** * User job id. * @@ -33,6 +40,46 @@ class CRM_Import_Forms extends CRM_Core_Form { */ protected $userJobID; + /** + * Name of the import mapping (civicrm_mapping). + * + * @var string + */ + protected $mappingName; + + /** + * The id of the saved mapping being updated. + * + * Note this may not be the same as the saved mapping being used to + * load data. Use the `getSavedMappingID` function to access & any + * extra logic can be added in there. + * + * @var int + */ + protected $savedMappingID; + + /** + * @param int $savedMappingID + * + * @return CRM_Import_Forms + */ + public function setSavedMappingID(int $savedMappingID): CRM_Import_Forms { + $this->savedMappingID = $savedMappingID; + return $this; + } + + /** + * Get the name of the type to be stored in civicrm_user_job.type_id. + * + * This should be overridden. + * + * @return string + */ + public function getUserJobType(): string { + CRM_Core_Error::deprecatedWarning('this function should be overridden'); + return ''; + } + /** * @return int|null */ @@ -137,6 +184,7 @@ class CRM_Import_Forms extends CRM_Core_Form { * @param string $fieldName * * @return mixed|null + * @throws \CRM_Core_Exception */ public function getSubmittedValue(string $fieldName) { if ($fieldName === 'dataSource') { @@ -155,6 +203,57 @@ class CRM_Import_Forms extends CRM_Core_Form { } + /** + * Get the template ID from the url, if available. + * + * Otherwise there are other possibilities... + * - it could already be saved to our UserJob. + * - on the DataSource form we could determine if from the savedMapping field + * (which will hold an ID that can be used to load it). We want to check this is + * coming from the POST (ie fresh) + * - on the MapField form it could be derived from the new mapping created from + * saveMapping + saveMappingName. + * + * @return int|null + * @noinspection PhpUnhandledExceptionInspection + * @noinspection PhpDocMissingThrowsInspection + */ + public function getTemplateID(): ?int { + if ($this->templateID === NULL) { + $this->templateID = CRM_Utils_Request::retrieve('template_id', 'Int', $this); + if ($this->templateID && $this->getTemplateJob()) { + return $this->templateID; + } + if ($this->getUserJobID()) { + $this->templateID = $this->getUserJob()['metadata']['template_id'] ?? NULL; + } + elseif (!empty($this->getSubmittedValue('savedMapping'))) { + if (!$this->getTemplateJob()) { + $this->createTemplateJob(); + } + } + } + return $this->templateID ?? NULL; + } + + /** + * @return string + * + * @throws \CRM_Core_Exception + */ + protected function getMappingName(): string { + if ($this->mappingName === NULL) { + $savedMappingID = $this->getSavedMappingID(); + if ($savedMappingID) { + $this->mappingName = Mapping::get(FALSE) + ->addWhere('id', '=', $savedMappingID) + ->execute() + ->first()['name']; + } + } + return $this->mappingName ?? ''; + } + /** * Get the available datasource. * @@ -263,10 +362,14 @@ class CRM_Import_Forms extends CRM_Core_Form { // We give the datasource a chance to clean up any tables it might have // created. If we are still using the same type of datasource (e.g still // an sql query - $oldDataSource = $this->getUserJobSubmittedValues()['dataSource']; - $oldDataSourceObject = new $oldDataSource($this->getUserJobID()); - $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ? $this->getSubmittedValues() : []; - $oldDataSourceObject->purge($newParams); + $oldDataSource = $this->getUserJobSubmittedValues()['dataSource'] ?? NULL; + if ($oldDataSource) { + // Absence of an old data source likely means a template has been used (hence + // the user job exists) - but templates don't have data sources - so nothing to flush. + $oldDataSourceObject = new $oldDataSource($this->getUserJobID()); + $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ? $this->getSubmittedValues() : []; + $oldDataSourceObject->purge($newParams); + } $this->updateUserJobMetadata('DataSource', []); } @@ -332,6 +435,7 @@ class CRM_Import_Forms extends CRM_Core_Form { * all forms. * * @return string[] + * @throws \CRM_Core_Exception */ protected function getSubmittableFields(): array { $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource'); @@ -379,6 +483,8 @@ class CRM_Import_Forms extends CRM_Core_Form { 'expires_date' => '+ 1 week', 'metadata' => [ 'submitted_values' => $this->getSubmittedValues(), + 'template_id' => $this->getTemplateID(), + 'Template' => ['mapping_id' => $this->getSavedMappingID()], ], ]) ->execute() @@ -387,6 +493,22 @@ class CRM_Import_Forms extends CRM_Core_Form { return $id; } + protected function createTemplateJob(): void { + if (!$this->getUserJobType()) { + // This could be hit in extensions while they transition. + CRM_Core_Error::deprecatedWarning('Classes should implement getUserJobType'); + return; + } + $this->templateID = UserJob::create(FALSE)->setValues([ + 'is_template' => 1, + 'created_id' => CRM_Core_Session::getLoggedInContactID(), + 'job_type' => $this->getUserJobType(), + 'status_id:name' => 'draft', + 'name' => 'import_' . $this->getMappingName(), + 'metadata' => ['submitted_values' => $this->getSubmittedValues()], + ])->execute()->first()['id']; + } + /** * @param string $key * @param array $data @@ -399,6 +521,14 @@ class CRM_Import_Forms extends CRM_Core_Form { $this->getUserJob()['metadata'], [$key => $data] ); + $this->getUserJob()['metadata'] = $metaData; + if ($this->isUpdateTemplateJob()) { + $this->updateTemplateUserJob($metaData); + } + // We likely don't need the empty check. A precaution against nulling it out by accident. + if (empty($metaData['template_id'])) { + $metaData['template_id'] = $this->templateID; + } UserJob::update(FALSE) ->addWhere('id', '=', $this->getUserJobID()) ->setValues(['metadata' => $metaData]) @@ -406,6 +536,17 @@ class CRM_Import_Forms extends CRM_Core_Form { $this->userJob['metadata'] = $metaData; } + /** + * Is the user wanting to update the template / mapping. + * + * @return bool + * + * @throws \CRM_Core_Exception + */ + protected function isUpdateTemplateJob(): bool { + return $this->getSubmittedValue('updateMapping') || $this->getSubmittedValue('saveMapping'); + } + /** * Get column headers for the datasource or empty array if none apply. * @@ -565,6 +706,18 @@ class CRM_Import_Forms extends CRM_Core_Form { return $summary; } + /** + * Get information about the user job parser. + * + * This is as per `CRM_Core_BAO_UserJob::getTypes()` + * + * @return array + */ + protected function getUserJobInfo(): array { + $importInformation = $this->getParser()->getUserJobInfo(); + return reset($importInformation); + } + /** * Get the fields available for import selection. * @@ -631,6 +784,7 @@ class CRM_Import_Forms extends CRM_Core_Form { * Get an instance of the parser class. * * @return \CRM_Contact_Import_Parser_Contact|\CRM_Contribute_Import_Parser_Contribution + * @throws \CRM_Core_Exception */ protected function getParser() { foreach (CRM_Core_BAO_UserJob::getTypes() as $jobType) { @@ -777,4 +931,56 @@ class CRM_Import_Forms extends CRM_Core_Form { ]); } + /** + * Get the UserJob Template, if it exists. + * + * @return array|null + * + * @throws \CRM_Core_Exception + */ + protected function getTemplateJob(): ?array { + $mappingName = $this->getMappingName(); + if (!$mappingName) { + return NULL; + } + $templateJob = UserJob::get(FALSE) + ->addWhere('name', '=', 'import_' . $mappingName) + ->addWhere('is_template', '=', TRUE) + ->execute()->first(); + $this->templateID = $templateJob['id']; + return $templateJob ?? NULL; + } + + /** + * @param array $metaData + * + * @throws \CRM_Core_Exception + * @throws \Civi\API\Exception\UnauthorizedException + */ + protected function updateTemplateUserJob(array $metaData): void { + if ($this->getTemplateID()) { + UserJob::update(FALSE) + ->addWhere('id', '=', $this->getTemplateID()) + ->setValues(['metadata' => $metaData, 'is_template' => TRUE]) + ->execute(); + } + elseif ($this->getMappingName()) { + $this->createTemplateJob(); + } + } + + /** + * Get the saved mapping ID being updated. + * + * @return int|null + */ + public function getSavedMappingID(): ?int { + if (!$this->savedMappingID) { + if (!empty($this->getUserJob()['metadata']['Template']['mapping_id'])) { + $this->savedMappingID = $this->getUserJob()['metadata']['Template']['mapping_id']; + } + } + return $this->savedMappingID; + } + } diff --git a/CRM/Report/Form/Contact/Summary.php b/CRM/Report/Form/Contact/Summary.php index 0d330b3ae7..4e658da660 100644 --- a/CRM/Report/Form/Contact/Summary.php +++ b/CRM/Report/Form/Contact/Summary.php @@ -16,7 +16,7 @@ */ class CRM_Report_Form_Contact_Summary extends CRM_Report_Form { - public $_summary = NULL; + public $_summary; protected $_emailField = FALSE; diff --git a/ext/civiimport/civiimport.php b/ext/civiimport/civiimport.php index d8df1ca609..b12f5a296c 100644 --- a/ext/civiimport/civiimport.php +++ b/ext/civiimport/civiimport.php @@ -208,7 +208,7 @@ function civiimport_civicrm_searchKitTasks(array &$tasks, bool $checkPermissions * Load the angular app for our form. * * @param string $formName - * @param \CRM_Core_Form|CRM_Contribute_Import_Form_MapField $form + * @param CRM_Contribute_Import_Form_MapField $form * * @throws \CRM_Core_Exception */ @@ -217,7 +217,7 @@ function civiimport_civicrm_buildForm(string $formName, $form) { // Add import-ui app Civi::service('angularjs.loader')->addModules('crmCiviimport'); $form->assignCiviimportVariables(); - $savedMappingID = (int) $form->getSubmittedValue('savedMapping'); + $savedMappingID = (int) $form->getSavedMappingID(); $savedMapping = []; if ($savedMappingID) { $savedMapping = Mapping::get()->addWhere('id', '=', $savedMappingID)->addSelect('id', 'name', 'description')->execute()->first(); @@ -245,5 +245,6 @@ function civiimport_civicrm_buildForm(string $formName, $form) { $form->assign('isOpenResultsInNewTab', TRUE); $form->assign('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/search', '', TRUE, '/display/Import_' . $form->getUserJobID() . '/Import_' . $form->getUserJobID() . '?_status=ERROR', FALSE)); $form->assign('allRowsUrl', CRM_Utils_System::url('civicrm/search', '', TRUE, '/display/Import_' . $form->getUserJobID() . '/Import_' . $form->getUserJobID(), FALSE)); + $form->assign('importedRowsUrl', CRM_Utils_System::url('civicrm/search', '', TRUE, '/display/Import_' . $form->getUserJobID() . '/Import_' . $form->getUserJobID() . '?_status=IMPORTED', FALSE)); } } diff --git a/templates/CRM/Contact/Import/Form/Summary.tpl b/templates/CRM/Contact/Import/Form/Summary.tpl index 19436b2c32..f0d6368f62 100644 --- a/templates/CRM/Contact/Import/Form/Summary.tpl +++ b/templates/CRM/Contact/Import/Form/Summary.tpl @@ -22,6 +22,10 @@ {ts}Import has completed successfully.{/ts} {/if}

+ {if $templateURL} +

+ {ts 1=$templateURL|smarty:nodefaults}You can re-use this import configuration here{/ts}

+ {/if} {if $unMatchCount}

@@ -61,7 +65,7 @@ {* Summary of Import Results (record counts) *} - + {if $unprocessedRowCount} @@ -100,7 +104,7 @@ - + {foreach from=$trackingSummary item="summaryRow"} diff --git a/templates/CRM/Import/Form/DataSource.tpl b/templates/CRM/Import/Form/DataSource.tpl index d62f2e7f99..2278273766 100644 --- a/templates/CRM/Import/Form/DataSource.tpl +++ b/templates/CRM/Import/Form/DataSource.tpl @@ -116,7 +116,7 @@ {/if} diff --git a/tests/phpunit/CRM/Contact/Import/Form/MapFieldTest.php b/tests/phpunit/CRM/Contact/Import/Form/MapFieldTest.php index 39dec4f75c..7e7e72495f 100644 --- a/tests/phpunit/CRM/Contact/Import/Form/MapFieldTest.php +++ b/tests/phpunit/CRM/Contact/Import/Form/MapFieldTest.php @@ -40,7 +40,7 @@ class CRM_Contact_Import_Form_MapFieldTest extends CiviUnitTestCase { * Delete any saved mapping config. */ public function tearDown(): void { - $this->quickCleanup(['civicrm_mapping', 'civicrm_mapping_field'], TRUE); + $this->quickCleanup(['civicrm_mapping', 'civicrm_mapping_field', 'civicrm_user_job', 'civicrm_queue'], TRUE); parent::tearDown(); } diff --git a/tests/phpunit/CRM/Import/DataSource/FormsTest.php b/tests/phpunit/CRM/Import/DataSource/FormsTest.php new file mode 100644 index 0000000000..512727db2a --- /dev/null +++ b/tests/phpunit/CRM/Import/DataSource/FormsTest.php @@ -0,0 +1,181 @@ +quickCleanup(['civicrm_user_job', 'civicrm_mapping', 'civicrm_mapping_field', 'civicrm_queue']); + parent::tearDown(); + } + + /** + * @throws \CRM_Core_Exception + */ + public function testLoadDataSourceSavedTemplate(): void { + // First do a basic submission, creating a Mapping and UserJob template in the process. + [$templateJob, $mapping] = $this->runImportSavingImportTemplate(); + + // Now try this template in in the url to load the defaults for DataSource. + $_REQUEST['template_id'] = $templateJob['id']; + $form = $this->getFormObject('CRM_Contribute_Import_Form_DataSource'); + $this->formController = $form->controller; + $form->buildForm(); + $defaults = $this->getFormDefaults($form); + // These next 2 fields should be loaded as defaults from the UserJob template. + $this->assertEquals('Organization', $defaults['contactType']); + $this->assertEquals([$mapping['id']], $defaults['savedMapping']); + } + + /** + * Test that when we Process the MapField form without updating the saved template it is still retained. + * + * This is important because if we use the BACK button we still want 'Update Mapping' + * to show. + */ + public function testSaveRetainingMappingID(): void { + // First do a basic submission, creating a Mapping and UserJob template in the process. + [, $mapping] = $this->runImportSavingImportTemplate(); + $this->formController = NULL; + + $dataSourceForm = $this->processForm('CRM_Contribute_Import_Form_DataSource', [ + 'contactType' => 'Organization', + 'savedMapping' => 1, + ]); + $userJobID = $dataSourceForm->getUserJobID(); + $this->processForm('CRM_Contribute_Import_Form_MapField', [ + 'savedMapping' => $mapping['id'], + 'contactType' => 'Organization', + 'mapper' => [['id'], ['source']], + ]); + + // Now we want to submit this form without updating the mapping used & make sure the mapping_id + // is still saved in the metadata. + /* @var CRM_Contribute_Import_Form_MapField $mapFieldForm */ + $mapFieldValues = [ + 'dataSource' => 'CRM_Import_DataSource_SQL', + 'sqlQuery' => 'SELECT id, source FROM civicrm_contact', + 'mapper' => [['id'], ['financial_type_id']], + ]; + $mapFieldForm = $this->getFormObject('CRM_Contribute_Import_Form_MapField', $mapFieldValues); + $mapFieldForm->buildForm(); + + $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->execute()->first(); + $this->assertEquals($mapping['id'], $userJob['metadata']['Template']['mapping_id']); + } + + /** + * Get the values specified as defaults for the form. + * + * I originally wanted to make this a public function on `CRM_Core_Form` + * but I think it might need to mature first. + */ + public function getFormDefaults($form): array { + $defaults = []; + if (!empty($form->_elementIndex)) { + foreach ($form->_elementIndex as $elementName => $elementIndex) { + $element = $form->_elements[$elementIndex]; + $defaults[$elementName] = $element->getValue(); + } + } + return $defaults; + } + + /** + * @param string $class + * @param array $formValues + * + * @return \CRM_Core_Form + */ + protected function processForm(string $class, array $formValues = []): CRM_Core_Form { + $form = $this->getImportForm($class, $formValues); + $form->buildForm(); + $form->mainProcess(); + return $form; + } + + /** + * Get some default values to use when we don't care. + * + * @return array + */ + protected function getDefaultValues(): array { + return [ + 'contactType' => 'Individual', + 'contactSubType' => '', + 'dataSource' => 'CRM_Import_DataSource_SQL', + 'sqlQuery' => 'SELECT id, source FROM civicrm_contact', + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, + 'mapper' => [['id'], ['source']], + ]; + } + + /** + * @param array $submittedValues + */ + protected function processContributionForms(array $submittedValues): void { + try { + $this->processForm('CRM_Contribute_Import_Form_DataSource', $submittedValues); + $this->processForm('CRM_Contribute_Import_Form_MapField', $submittedValues); + $this->processForm('CRM_Contribute_Import_Form_Preview', $submittedValues); + } + catch (CRM_Core_Exception_PrematureExitException $e) { + // We expect this to happen as it re-directs to the queue runner. + } + } + + /** + * @param string $class + * @param array $formValues + * + * @return \CRM_Core_Form + */ + protected function getImportForm(string $class, array $formValues = []): CRM_Core_Form { + $formValues = array_merge($this->getDefaultValues(), $formValues); + return $this->getFormObject($class, $formValues); + } + + /** + * @return array + */ + protected function runImportSavingImportTemplate(): array { + $this->processContributionForms([ + 'saveMapping' => 1, + 'saveMappingName' => 'mapping', + 'contactType' => 'Organization', + ]); + + // Check that a template job and a mapping have been created. + $templateJob = UserJob::get() + ->addWhere('is_template', '=', 1) + ->execute() + ->first();; + $this->assertNotEmpty($templateJob); + $this->assertArrayNotHasKey('table_name', $templateJob['metadata']['DataSource']); + $mapping = Mapping::get() + ->addWhere('name', '=', substr($templateJob['name'], 7)) + ->execute() + ->first(); + $this->assertNotEmpty($mapping); + // Reset the formController so this doesn't leak into further tests. + $this->formController = NULL; + return [$templateJob, $mapping]; + } + +} diff --git a/tests/phpunit/CiviTest/CiviUnitTestCase.php b/tests/phpunit/CiviTest/CiviUnitTestCase.php index 5c2ccb2cf4..e90decf50d 100644 --- a/tests/phpunit/CiviTest/CiviUnitTestCase.php +++ b/tests/phpunit/CiviTest/CiviUnitTestCase.php @@ -471,9 +471,10 @@ class CiviUnitTestCase extends PHPUnit\Framework\TestCase { $this->unsetExtensionSystem(); $this->assertEquals([], CRM_Core_DAO::$_nullArray); $this->assertEquals(NULL, CRM_Core_DAO::$_nullObject); - // Ensure the destruct runs by unsetting it. Also, unsetting - // classes frees memory as they are not otherwise unset until the - // very end. + // Setting large properties to NULL here ensures memory is released as each + // test class is held in memory until the very end. + $this->formController = NULL; + // Ensure the destruct runs by unsetting the Mutt. unset($this->mut); parent::tearDown(); } @@ -3183,8 +3184,15 @@ class CiviUnitTestCase extends PHPUnit\Framework\TestCase { case 'CRM_Contribute_Import_Form_DataSource': case 'CRM_Contribute_Import_Form_MapField': case 'CRM_Contribute_Import_Form_Preview': - $form->controller = new CRM_Contribute_Import_Controller(); - $form->controller->setStateMachine(new CRM_Core_StateMachine($form->controller)); + if ($this->formController) { + // Add to the existing form controller. + $form->controller = $this->formController; + } + else { + $form->controller = new CRM_Contribute_Import_Controller(); + $form->controller->setStateMachine(new CRM_Core_StateMachine($form->controller)); + $this->formController = $form->controller; + } // The submitted values should be set on one or the other of the forms in the flow. // For test simplicity we set on all rather than figuring out which ones go where.... $_SESSION['_' . $form->controller->_name . '_container']['values']['DataSource'] = $formValues; -- 2.25.1
{ts}Total Rows{/ts}{$totalRowCount}{if $allRowsUrl} {$totalRowCount}{else}{$totalRowCount}{/if} {ts}Total number of rows in the imported data.{/ts}
{ts}Total Rows Imported{/ts}{$importedRowCount}{if $importedRowsUrl} {$importedRowCount}{else}{$importedRowCount}{/if} {ts}Total number of primary records created or modified during the import.{/ts}
{$form.savedMapping.html}
- {ts}If you want to use a previously saved import field mapping - select it here.{/ts} + {if !$form.savedMapping.frozen}{ts}If you want to use a previously saved import field mapping - select it here.{/ts}{/if}