From 3592a5e4098c9d14b2577a0f6eee5c6c7d471501 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Fri, 3 Jun 2022 02:14:34 +1200 Subject: [PATCH] Use UI Queue runner for import Additional minor code simplification Tear down fix Maybe the var name needs to match in the CI php version --- CRM/Contact/Import/Form/Preview.php | 65 +++--------- CRM/Contact/Import/Form/Summary.php | 13 +-- CRM/Contact/Import/Parser/Contact.php | 99 +------------------ CRM/Core/xml/Menu/Import.xml | 8 ++ CRM/Import/Parser.php | 23 +++++ templates/CRM/Contact/Import/Form/Preview.tpl | 1 - .../CRM/Contact/Import/Parser/ContactTest.php | 16 ++- 7 files changed, 65 insertions(+), 160 deletions(-) diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index 37ad7552ec..57ea179672 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -23,25 +23,6 @@ use Civi\Api4\Tag; */ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { - /** - * Whether USPS validation should be disabled during import. - * - * @var bool - */ - protected $_disableUSPS; - - /** - * Set variables up before form is built. - * - * @throws \API_Exception - * @throws \CRM_Core_Exception - */ - public function preProcess() { - parent::preProcess(); - $this->_disableUSPS = $this->getSubmittedValue('disableUSPS'); - $this->setStatusUrl(); - } - /** * Build the form object. */ @@ -92,7 +73,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { * @param $files * @param self $self * - * @return array + * @return array|bool * list of errors to be posted back to the form */ public static function formRule($fields, $files, $self) { @@ -210,43 +191,19 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { CRM_ACL_BAO_Cache::deleteContactCacheEntry($userID); } - CRM_Utils_Address_USPS::disable($this->_disableUSPS); + CRM_Utils_Address_USPS::disable($this->getSubmittedValue('disableUSPS')); // run the import - $this->_parser = $this->getParser(); - $this->_parser->run( - [], - CRM_Import_Parser::MODE_IMPORT, - $this->get('statusID') - ); - - // Clear all caches, forcing any searches to recheck the ACLs or group membership as the import - // may have changed it. - CRM_Contact_BAO_Contact_Utils::clearContactCaches(TRUE); - - // check if there is any error occurred - // @todo - it's really unclear that this error code should still exist... - $errorStack = CRM_Core_Error::singleton(); - $errors = $errorStack->getErrors(); - $errorMessage = []; - - if (is_array($errors)) { - foreach ($errors as $key => $value) { - $errorMessage[] = $value['message']; - } - - // there is no fileName since this is a sql import - // so fudge it - $config = CRM_Core_Config::singleton(); - $errorFile = $config->uploadDir . "sqlImport.error.log"; - if ($fd = fopen($errorFile, 'w')) { - fwrite($fd, implode('\n', $errorMessage)); - } - fclose($fd); - - $this->set('errorFile', $errorFile); - } + $parser = $this->getParser(); + $parser->queue(); + $queue = Civi::queue('user_job_' . $this->getUserJobID()); + $runner = new CRM_Queue_Runner([ + 'queue' => $queue, + 'errorMode' => CRM_Queue_Runner::ERROR_ABORT, + 'onEndUrl' => CRM_Utils_System::url('civicrm/import/contact/summary', ['user_job_id' => $this->getUserJobID(), 'reset' => 1]), + ]); + $runner->runAllViaWeb(); } /** diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php index 33d02af4c6..c8adb53bfe 100644 --- a/CRM/Contact/Import/Form/Summary.php +++ b/CRM/Contact/Import/Form/Summary.php @@ -15,6 +15,8 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\UserJob; + /** * This class summarizes the import results. */ @@ -27,16 +29,15 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary { * @throws \CRM_Core_Exception */ public function preProcess() { - // @todo - totally unclear that this errorFile could ever be set / render. - // Probably it can go. - $this->assign('errorFile', $this->get('errorFile')); - $onDuplicate = $this->getSubmittedValue('onDuplicate'); + $userJobID = CRM_Utils_Request::retrieve('user_job_id', 'String', $this, TRUE); + $userJob = UserJob::get(TRUE)->addWhere('id', '=', $userJobID)->execute()->first(); + $onDuplicate = $userJob['metadata']['submitted_values']['onDuplicate']; $this->assign('dupeError', FALSE); - if ($onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { + if ($onDuplicate === CRM_Import_Parser::DUPLICATE_UPDATE) { $this->assign('dupeActionString', ts('These records have been updated with the imported data.')); } - elseif ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) { + elseif ($onDuplicate === CRM_Import_Parser::DUPLICATE_FILL) { $this->assign('dupeActionString', ts('These records have been filled in with the imported data.')); } else { diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index e51271549b..f72d38d42e 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -45,13 +45,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { */ protected $_lineCount; - /** - * Array of successfully imported related contact id's - * - * @var array - */ - protected $_newRelatedContacts; - protected $_tableName; /** @@ -73,20 +66,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { */ protected $relationshipLabels = []; - /** - * On duplicate - * - * @var int - */ - public $_onDuplicate; - - /** - * Dedupe rule group id to use if set - * - * @var int - */ - public $_dedupeRuleGroupID = NULL; - /** * Addresses that failed to parse. * @@ -182,16 +161,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { return CRM_Import_Parser::VALID; } - /** - * Get Array of all the fields that could potentially be part - * import process - * - * @return array - */ - public function getAllFields() { - return $this->_fields; - } - /** * Handle the values in import mode. * @@ -251,7 +220,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { //fixed CRM-4148 //now we create new contact in update/fill mode also. - $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $params['id'] ?? NULL, TRUE, $this->_dedupeRuleGroupID); + $newContact = $this->createContact($formatted, $contactFields, $onDuplicate, $params['id'] ?? NULL, TRUE, $this->getSubmittedValue('dedupe_rule_id')); $this->createdContacts[$newContact->id] = $contactID = $newContact->id; if ($contactID) { @@ -1200,70 +1169,6 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { $this->setImportableFieldsMetadata($this->getContactImportMetadata()); } - /** - * Run import. - * - * @param array $mapper Mapping as entered on MapField form. - * e.g [['first_name']['email', 1]]. - * {@see \CRM_Contact_Import_Parser_Contact::getMappingFieldFromMapperInput} - * @param int $mode - * @param int $statusID - * - * @return mixed - * @throws \API_Exception|\CRM_Core_Exception - */ - public function run( - $mapper = [], - $mode = self::MODE_PREVIEW, - $statusID = NULL - ) { - - // TODO: Make the timeout actually work - $this->_onDuplicate = $onDuplicate = $this->getSubmittedValue('onDuplicate'); - $this->_dedupeRuleGroupID = $this->getSubmittedValue('dedupe_rule_id'); - // Since $this->_contactType is still being called directly do a get call - // here to make sure it is instantiated. - $this->getContactType(); - $this->getContactSubType(); - // Reset user job in case to null in case it was loaded prior to the job being complete. - $this->userJob = NULL; - - $this->init(); - - $this->_rowCount = 0; - $this->_totalCount = 0; - - if ($statusID) { - $this->progressImport($statusID); - $startTimestamp = $currTimestamp = $prevTimestamp = time(); - } - $dataSource = $this->getDataSourceObject(); - $totalRowCount = $dataSource->getRowCount(['new']); - $dataSource->setStatuses(['new']); - - while ($row = $dataSource->getRow()) { - $values = array_values($row); - $this->_rowCount++; - - $this->_totalCount++; - - try { - $this->import($onDuplicate, $values); - } - catch (CiviCRM_API3_Exception $e) { - // When we catch errors here we are not adding to the errors array - mostly - // because that will become obsolete once https://github.com/civicrm/civicrm-core/pull/23292 - // is merged and this will replace it as the main way to handle errors (ie. update the table - // and move on). - $this->setImportStatus((int) $values[count($values) - 1], 'ERROR', $e->getMessage()); - } - if ($statusID && (($this->_rowCount % 50) == 0)) { - $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); - } - } - $this->doPostImportActions(); - } - /** * @param string $name * @param $title @@ -2139,7 +2044,7 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { CRM_Import_Parser::DUPLICATE => 'DUPLICATE', CRM_Import_Parser::ERROR => 'ERROR', CRM_Import_Parser::NO_MATCH => 'invalid_no_match', - ][$outcome]; + ][$outcome] ?? 'ERROR'; } } diff --git a/CRM/Core/xml/Menu/Import.xml b/CRM/Core/xml/Menu/Import.xml index 33d935c634..7942d3e496 100644 --- a/CRM/Core/xml/Menu/Import.xml +++ b/CRM/Core/xml/Menu/Import.xml @@ -17,6 +17,14 @@ CRM_Contact_Import_Controller 410 + + civicrm/import/contact/summary + Import Contacts + import contacts,access CiviCRM + 1 + CRM_Contact_Import_Form_Summary + 410 + civicrm/import/outcome Import results diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index f072271442..2421640b8a 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -646,6 +646,28 @@ abstract class CRM_Import_Parser { UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute(); } + public function queue() { + $dataSource = $this->getDataSourceObject(); + $totalRowCount = $totalRows = $dataSource->getRowCount(['new']); + $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']); + $offset = 0; + $batchSize = 5; + while ($totalRows > 0) { + if ($totalRows < $batchSize) { + $batchSize = $totalRows; + } + $task = new CRM_Queue_Task( + [get_class($this), 'runImport'], + ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize], + ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount]) + ); + $queue->createItem($task); + $totalRows -= $batchSize; + $offset += $batchSize; + } + + } + /** * Add imported contacts to groups. * @@ -1675,6 +1697,7 @@ abstract class CRM_Import_Parser { $parserClass = $userJobType['class']; } } + /* @var \CRM_Import_Parser $parser */ $parser = new $parserClass(); $parser->setUserJobID($userJobID); // Not sure if we still need to init.... diff --git a/templates/CRM/Contact/Import/Form/Preview.tpl b/templates/CRM/Contact/Import/Form/Preview.tpl index 77b28e9b6f..47283e4dff 100644 --- a/templates/CRM/Contact/Import/Form/Preview.tpl +++ b/templates/CRM/Contact/Import/Form/Preview.tpl @@ -27,7 +27,6 @@

{ts}Click 'Import Now' if you are ready to proceed.{/ts}

{include file="CRM/common/formButtons.tpl" location="top"}
-{include file="CRM/common/importProgress.tpl"}
{* Summary Preview (record counts) *} diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php index 02bdc43963..b665d9986b 100644 --- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php +++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php @@ -53,7 +53,7 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { * Tear down after test. */ public function tearDown(): void { - $this->quickCleanup(['civicrm_address', 'civicrm_phone', 'civicrm_openid', 'civicrm_email', 'civicrm_user_job', 'civicrm_relationship', 'civicrm_im', 'civicrm_website'], TRUE); + $this->quickCleanup(['civicrm_address', 'civicrm_phone', 'civicrm_openid', 'civicrm_email', 'civicrm_user_job', 'civicrm_relationship', 'civicrm_im', 'civicrm_website', 'civicrm_queue', 'civicrm_queue_item'], TRUE); RelationshipType::delete()->addWhere('name_a_b', '=', 'Dad to')->execute(); ContactType::delete()->addWhere('name', '=', 'baby')->execute(); parent::tearDown(); @@ -2040,7 +2040,18 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { $form = $this->getFormObject('CRM_Contact_Import_Form_Preview', $submittedValues); $form->setUserJobID($userJobID); $form->buildForm(); - $form->postProcess(); + + try { + $form->postProcess(); + } + catch (CRM_Core_Exception_PrematureExitException $e) { + $queue = Civi::queue('user_job_' . $userJobID); + $runner = new CRM_Queue_Runner([ + 'queue' => $queue, + 'errorMode' => CRM_Queue_Runner::ERROR_ABORT, + ]); + $runner->runAll(); + } } /** @@ -2075,6 +2086,7 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase { } } } + UserJob::delete()->addWhere('id', '=', $parser->getUserJobID())->execute(); } /** -- 2.25.1