public function buildQuickForm() {
$this->assign('urlPath', 'civicrm/import/datasource');
- $this->assign('urlPathVar', 'snippet=4');
+ $this->assign('urlPathVar', 'snippet=4&user_job_id=' . $this->get('user_job_id'));
$this->add('select', 'dataSource', ts('Data Source'), $this->getDataSources(), TRUE,
['onchange' => 'buildDataSourceFormBlock(this.value);']
* Call the DataSource's postProcess method.
*
* @throws \CRM_Core_Exception
+ * @throws \API_Exception
*/
public function postProcess() {
$this->controller->resetPage('MapField');
-
+ if (!$this->getUserJobID()) {
+ $this->createUserJob();
+ }
+ else {
+ $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues());
+ }
// Setup the params array
$this->_params = $this->controller->exportValues($this->_name);
$parser = new CRM_Contact_Import_Parser_Contact($mapper);
$parser->setMaxLinesToProcess(100);
+ $parser->setUserJobID($this->getUserJobID());
$parser->run($importTableName,
$mapper,
CRM_Import_Parser::MODE_MAPFIELD,
* @throws \CRM_Core_Exception
*/
private function instantiateDataSource(): void {
- $dataSourceName = $this->getDataSourceClassName();
- $dataSource = new $dataSourceName();
+ $dataSource = $this->getDataSourceObject();
// Get the PEAR::DB object
$dao = new CRM_Core_DAO();
$db = $dao->getDatabaseConnection();
$dataSource->postProcess($this->_params, $db, $this);
+ $this->updateUserJobMetadata('DataSource', $dataSource->getDataSourceMetadata());
}
/**
],
[
'id' => 2,
+ 'name' => 'draft',
+ 'label' => ts('Draft'),
+ ],
+ [
+ 'id' => 3,
'name' => 'scheduled',
'label' => ts('Scheduled'),
],
[
- 'id' => 3,
+ 'id' => 4,
'name' => 'in_progress',
'label' => ts('In Progress'),
],
* @copyright CiviCRM LLC https://civicrm.org/licensing
*/
+use Civi\Api4\UserJob;
+
/**
* This class defines the DataSource interface but must be subclassed to be
* useful.
*/
abstract class CRM_Import_DataSource {
+ /**
+ * Class constructor.
+ *
+ * @param int|null $userJobID
+ */
+ public function __construct(int $userJobID = NULL) {
+ if ($userJobID) {
+ $this->setUserJobID($userJobID);
+ }
+ }
+
+ /**
+ * Form fields declared for this datasource.
+ *
+ * @var string[]
+ */
+ protected $submittableFields = [];
+
+ /**
+ * User job id.
+ *
+ * This is the primary key of the civicrm_user_job table which is used to
+ * track the import.
+ *
+ * @var int
+ */
+ protected $userJobID;
+
+ /**
+ * @return int|null
+ */
+ public function getUserJobID(): ?int {
+ return $this->userJobID;
+ }
+
+ /**
+ * Set user job ID.
+ *
+ * @param int $userJobID
+ */
+ public function setUserJobID(int $userJobID): void {
+ $this->userJobID = $userJobID;
+ }
+
+ /**
+ * User job details.
+ *
+ * This is the relevant row from civicrm_user_job.
+ *
+ * @var array
+ */
+ protected $userJob;
+
+ /**
+ * Get User Job.
+ *
+ * API call to retrieve the userJob row.
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ */
+ protected function getUserJob(): array {
+ if (!$this->userJob) {
+ $this->userJob = UserJob::get()
+ ->addWhere('id', '=', $this->getUserJobID())
+ ->execute()
+ ->first();
+ }
+ return $this->userJob;
+ }
+
+ /**
+ * Generated metadata relating to the the datasource.
+ *
+ * This is values that are computed within the DataSource class and
+ * which are stored in the userJob metadata in the DataSource key - eg.
+ *
+ * ['table_name' => $]
+ *
+ * Will be in the user_job.metadata field encoded into the json like
+ *
+ * `{'DataSource' : ['table_name' => $], 'submitted_values' : .....}`
+ *
+ * @var array
+ */
+ protected $dataSourceMetadata = [];
+
+ /**
+ * @return array
+ */
+ public function getDataSourceMetadata(): array {
+ return $this->dataSourceMetadata;
+ }
+
+ /**
+ * Get the fields declared for this datasource.
+ *
+ * @return string[]
+ */
+ public function getSubmittableFields(): array {
+ return $this->submittableFields;
+ }
+
/**
* Provides information about the data source.
*
const
NUM_ROWS_TO_INSERT = 100;
+ /**
+ * Form fields declared for this datasource.
+ *
+ * @var string[]
+ */
+ protected $submittableFields = ['skipColumnHeader', 'uploadField'];
+
/**
* Provides information about the data source.
*
CRM_Utils_Array::value('fieldSeparator', $params, ',')
);
- $form->set('originalColHeader', CRM_Utils_Array::value('original_col_header', $result));
+ $form->set('originalColHeader', CRM_Utils_Array::value('column_headers', $result));
$form->set('importTableName', $result['import_table_name']);
+ $this->dataSourceMetadata = [
+ 'table_name' => $result['import_table_name'],
+ 'column_headers' => $result['column_headers'] ?? NULL,
+ ];
}
/**
// create the column names from the CSV header or as col_0, col_1, etc.
if ($headers) {
//need to get original headers.
- $result['original_col_header'] = $firstrow;
+ $result['column_headers'] = $firstrow;
$strtolower = function_exists('mb_strtolower') ? 'mb_strtolower' : 'strtolower';
$columns = array_map($strtolower, $firstrow);
//get the import tmp table name.
$result['import_table_name'] = $tableName;
-
return $result;
}
*/
class CRM_Import_DataSource_SQL extends CRM_Import_DataSource {
+ /**
+ * Form fields declared for this datasource.
+ *
+ * @var string[]
+ */
+ protected $submittableFields = ['sqlQuery'];
+
/**
* Provides information about the data source.
*
);
$form->set('importTableName', $importJob->getTableName());
+ $this->dataSourceMetadata = [
+ 'table_name' => $importJob->getTableName(),
+ ];
}
}
$dataSourcePath = explode('_', $this->getDataSourceClassName());
$templateFile = 'CRM/Contact/Import/Form/' . $dataSourcePath[3] . '.tpl';
$this->assign('dataSourceFormTemplateFile', $templateFile ?? NULL);
+ if (CRM_Utils_Request::retrieveValue('user_job_id', 'Integer')) {
+ $this->setUserJobID(CRM_Utils_Request::retrieveValue('user_job_id', 'Integer'));
+ }
}
/**
* Build the form object.
+ *
+ * @throws \CRM_Core_Exception
*/
public function buildQuickForm(): void {
$this->buildDataSourceFields();
}
+ /**
+ * Set defaults.
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ public function setDefaultValues() {
+ $defaults = [];
+ if ($this->userJobID) {
+ foreach ($this->getDataSourceFields() as $fieldName) {
+ $defaults[$fieldName] = $this->getSubmittedValue($fieldName);
+ }
+ }
+ return $defaults;
+ }
+
+ /**
+ * Get the submitted value, as saved in the user job.
+ *
+ * This form is not in the same flow as the DataSource but
+ * the value we want is saved to the userJob so load it from there.
+ *
+ * @param string $fieldName
+ *
+ * @return mixed|null
+ * @throws \API_Exception
+ */
+ public function getSubmittedValue(string $fieldName) {
+ $userJob = $this->getUserJob();
+ return $userJob['metadata']['submitted_values'][$fieldName];
+ }
+
}
* @copyright CiviCRM LLC https://civicrm.org/licensing
*/
+use Civi\Api4\UserJob;
+
/**
* This class helps the forms within the import flow access submitted & parsed values.
*/
class CRM_Import_Forms extends CRM_Core_Form {
/**
- * Get the submitted value, accessing it from whatever form in the flow it is submitted on.
+ * User job id.
+ *
+ * This is the primary key of the civicrm_user_job table which is used to
+ * track the import.
+ *
+ * @var int
+ */
+ protected $userJobID;
+
+ /**
+ * @return int|null
+ */
+ public function getUserJobID(): ?int {
+ if (!$this->userJobID && $this->get('user_job_id')) {
+ $this->userJobID = $this->get('user_job_id');
+ }
+ return $this->userJobID;
+ }
+
+ /**
+ * Set user job ID.
+ *
+ * @param int $userJobID
+ */
+ public function setUserJobID(int $userJobID): void {
+ $this->userJobID = $userJobID;
+ // This set allows other forms in the flow ot use $this->get('user_job_id').
+ $this->set('user_job_id', $userJobID);
+ }
+
+ /**
+ * User job details.
+ *
+ * This is the relevant row from civicrm_user_job.
+ *
+ * @var array
+ */
+ protected $userJob;
+
+ /**
+ * Get User Job.
+ *
+ * API call to retrieve the userJob row.
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ */
+ protected function getUserJob(): array {
+ if (!$this->userJob) {
+ $this->userJob = UserJob::get()
+ ->addWhere('id', '=', $this->getUserJobID())
+ ->execute()
+ ->first();
+ }
+ return $this->userJob;
+ }
+
+ /**
+ * Get submitted values stored in the user job.
+ *
+ * @return array
+ * @throws \API_Exception
+ */
+ protected function getUserJobSubmittedValues(): array {
+ return $this->getUserJob()['metadata']['submitted_values'];
+ }
+
+ /**
+ * Fields that may be submitted on any form in the flow.
+ *
+ * @var string[]
+ */
+ protected $submittableFields = [
+ // Skip column header is actually a field that would be added from the
+ // datasource - but currently only in contact, it is always there for
+ // other imports, ditto uploadFile.
+ 'skipColumnHeader' => 'DataSource',
+ 'fieldSeparator' => 'DataSource',
+ 'uploadFile' => 'DataSource',
+ 'contactType' => 'DataSource',
+ 'dateFormats' => 'DataSource',
+ 'savedMapping' => 'DataSource',
+ 'dataSource' => 'DataSource',
+ ];
+
+ /**
+ * Get the submitted value, accessing it from whatever form in the flow it is
+ * submitted on.
+ *
* @param string $fieldName
*
* @return mixed|null
+ * @throws \CRM_Core_Exception
*/
public function getSubmittedValue(string $fieldName) {
- $mappedValues = [
- 'skipColumnHeader' => 'DataSource',
- 'fieldSeparator' => 'DataSource',
- 'uploadFile' => 'DataSource',
- 'contactType' => 'DataSource',
- 'dateFormats' => 'DataSource',
- 'savedMapping' => 'DataSource',
- 'dataSource' => 'DataSource',
- ];
+ if ($fieldName === 'dataSource') {
+ // Hard-coded handling for DataSource as it affects the contents of
+ // getSubmittableFields and can cause a loop.
+ return $this->controller->exportValue('DataSource', 'dataSource');
+ }
+ $mappedValues = $this->getSubmittableFields();
if (array_key_exists($fieldName, $mappedValues)) {
return $this->controller->exportValue($mappedValues[$fieldName], $fieldName);
}
}
+ /**
+ * Get values submitted on any form in the multi-page import flow.
+ *
+ * @return array
+ */
+ public function getSubmittedValues(): array {
+ $values = [];
+ foreach (array_keys($this->getSubmittableFields()) as $key) {
+ $values[$key] = $this->getSubmittedValue($key);
+ }
+ return $values;
+ }
+
/**
* Get the available datasource.
*
* @throws \CRM_Core_Exception
*/
protected function buildDataSourceFields(): void {
+ $dataSourceClass = $this->getDataSourceObject();
+ if ($dataSourceClass) {
+ $dataSourceClass->buildQuickForm($this);
+ }
+ }
+
+ /**
+ * Get the relevant datasource object.
+ *
+ * @return \CRM_Import_DataSource|null
+ *
+ * @throws \CRM_Core_Exception
+ */
+ protected function getDataSourceObject(): ?CRM_Import_DataSource {
+ $className = $this->getDataSourceClassName();
+ if ($className) {
+ /* @var CRM_Import_DataSource $dataSource */
+ return new $className($this->getUserJobID());
+ }
+ return NULL;
+ }
+
+ /**
+ * Allow the datasource class to add fields.
+ *
+ * This is called as a snippet in DataSourceConfig and
+ * also from DataSource::buildForm to add the fields such
+ * that quick form picks them up.
+ *
+ * @throws \CRM_Core_Exception
+ */
+ protected function getDataSourceFields(): array {
$className = $this->getDataSourceClassName();
if ($className) {
+ /* @var CRM_Import_DataSource $dataSourceClass */
$dataSourceClass = new $className();
- $dataSourceClass->buildQuickForm($this);
+ return $dataSourceClass->getSubmittableFields();
}
+ return [];
}
/**
return 'CRM_Import_DataSource_CSV';
}
+ /**
+ * Get the fields that can be submitted in the Import form flow.
+ *
+ * These could be on any form in the flow & are accessed the same way from
+ * all forms.
+ *
+ * @return string[]
+ * @throws \CRM_Core_Exception
+ */
+ protected function getSubmittableFields(): array {
+ $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource');
+ return array_merge($this->submittableFields, $dataSourceFields);
+ }
+
+ /**
+ * Create a user job to track the import.
+ *
+ * @return int
+ *
+ * @throws \API_Exception
+ */
+ protected function createUserJob(): int {
+ $id = UserJob::create(FALSE)
+ ->setValues([
+ 'created_id' => CRM_Core_Session::getLoggedInContactID(),
+ 'type_id:name' => 'contact_import',
+ 'status_id:name' => 'draft',
+ // This suggests the data could be cleaned up after this.
+ 'expires_date' => '+ 1 week',
+ 'metadata' => [
+ 'submitted_values' => $this->getSubmittedValues(),
+ ],
+ ])
+ ->execute()
+ ->first()['id'];
+ $this->setUserJobID($id);
+ return $id;
+ }
+
+ /**
+ * @param string $key
+ * @param array $data
+ *
+ * @throws \API_Exception
+ * @throws \Civi\API\Exception\UnauthorizedException
+ */
+ protected function updateUserJobMetadata(string $key, array $data): void {
+ $metaData = array_merge(
+ $this->getUserJob()['metadata'],
+ [$key => $data]
+ );
+ UserJob::update(FALSE)
+ ->addWhere('id', '=', $this->getUserJobID())
+ ->setValues(['metadata' => $metaData])
+ ->execute();
+ $this->userJob['metadata'] = $metaData;
+ }
+
}
+--------------------------------------------------------------------+
*/
+use Civi\Api4\UserJob;
+
/**
*
* @package CRM
const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
+ /**
+ * User job id.
+ *
+ * This is the primary key of the civicrm_user_job table which is used to
+ * track the import.
+ *
+ * @var int
+ */
+ protected $userJobID;
+
+ /**
+ * @return int|null
+ */
+ public function getUserJobID(): ?int {
+ return $this->userJobID;
+ }
+
+ /**
+ * Set user job ID.
+ *
+ * @param int $userJobID
+ */
+ public function setUserJobID(int $userJobID): void {
+ $this->userJobID = $userJobID;
+ }
+
+ /**
+ * Get User Job.
+ *
+ * API call to retrieve the userJob row.
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ */
+ protected function getUserJob(): array {
+ return UserJob::get()
+ ->addWhere('id', '=', $this->getUserJobID())
+ ->execute()
+ ->first();
+ }
+
/**
* Total number of non empty lines
* @var int
function buildDataSourceFormBlock(dataSource)
{
- var dataUrl = {/literal}"{crmURL p=$urlPath h=0 q=$urlPathVar}"{literal};
+ var dataUrl = {/literal}"{crmURL p=$urlPath h=0 q=$urlPathVar|smarty:nodefaults}"{literal};
if (!dataSource ) {
var dataSource = cj("#dataSource").val();
* File for the CRM_Contact_Import_Form_DataSourceTest class.
*/
+use Civi\Api4\UserJob;
+
/**
* Test contact import datasource.
*
*/
class CRM_Contact_Import_Form_DataSourceTest extends CiviUnitTestCase {
+ /**
+ * Post test cleanup.
+ */
+ public function tearDown(): void {
+ $this->quickCleanup(['civicrm_user_job']);
+ parent::tearDown();
+ }
+
/**
* Test the form loads without error / notice and mappings are assigned.
*
* (Added in conjunction with fixed noting on mapping assignment).
*/
- public function testBuildForm() {
+ public function testBuildForm(): void {
$this->callAPISuccess('Mapping', 'create', ['name' => 'Well dressed ducks', 'mapping_type_id' => 'Import Contact']);
$form = $this->getFormObject('CRM_Contact_Import_Form_DataSource');
$form->buildQuickForm();
}
/**
- * Check for (lack of) sql errors on sql import post process.
+ * Test sql and csv data-sources load and save user jobs.
+ *
+ * This test mimics a scenario where the form is submitted more than once
+ * and the user_job is updated to reflect the new data source.
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ * @throws \Civi\API\Exception\UnauthorizedException
*/
- public function testSQLSource() {
+ public function testDataSources(): void {
+ $this->createLoggedInUser();
$this->callAPISuccess('Mapping', 'create', ['name' => 'Well dressed ducks', 'mapping_type_id' => 'Import Contact']);
- /** @var CRM_Import_DataSource_SQL $form */
- $form = $this->getFormObject('CRM_Import_DataSource_SQL', [], 'SQL');
- $coreForm = $this->getFormObject('CRM_Core_Form');
- $db = NULL;
- $params = ['sqlQuery' => 'SELECT 1 as id'];
- $form->postProcess($params, $db, $coreForm);
+
+ $sqlFormValues = [
+ 'dataSource' => 'CRM_Import_DataSource_SQL',
+ 'sqlQuery' => 'SELECT "bob" as first_name FROM civicrm_option_value LIMIT 5',
+ ];
+ $form = $this->submitDataSourceForm($sqlFormValues);
+ $userJobID = $form->getUserJobID();
+ // Load the user job, using TRUE so permissions apply.
+ $userJob = UserJob::get(TRUE)
+ ->addWhere('id', '=', $userJobID)
+ ->addSelect('metadata')
+ ->execute()->first();
+ // Submitted values should be stored in the user job.
+ // There are some null values in the submitted_values array - we can
+ // filter these out as we have not passed in all possible values.
+ $this->assertEquals($sqlFormValues, array_filter($userJob['metadata']['submitted_values']));
+
+ // The user job holds the name of the table - which should have 5 rows of bob.
+ $this->assertNotEmpty($userJob['metadata']['DataSource']['table_name']);
+ $sqlTableName = $userJob['metadata']['DataSource']['table_name'];
+ $this->assertEquals(5, CRM_Core_DAO::singleValueQuery(
+ 'SELECT count(*) FROM ' . $sqlTableName
+ . " WHERE first_name = 'Bob'"
+ ));
+
+ // Now we imitate the scenario where the user goes back and
+ // re-submits the form selecting the csv datasource.
+ $csvFormValues = [
+ 'dataSource' => 'CRM_Import_DataSource_CSV',
+ 'skipColumnHeader' => 1,
+ 'uploadFile' => [
+ 'name' => __DIR__ . '/data/yogi.csv',
+ 'type' => 'text/csv',
+ ],
+ ];
+ // Mimic form re-submission with new values.
+ $_SESSION['_' . $form->controller->_name . '_container']['values']['DataSource'] = $csvFormValues;
+ $form->buildForm();
+ $form->postProcess();
+ // The user job id should not have changed.
+ $this->assertEquals($userJobID, $form->getUserJobID());
+
+ $userJob = UserJob::get(TRUE)
+ ->addWhere('id', '=', $form->getUserJobID())
+ ->addSelect('metadata')
+ ->execute()->first();
+ // Submitted values should be updated in the user job.
+ $this->assertEquals($csvFormValues, array_filter($userJob['metadata']['submitted_values']));
+
+ $csvTableName = $userJob['metadata']['DataSource']['table_name'];
+ $this->assertEquals(1, CRM_Core_DAO::singleValueQuery(
+ 'SELECT count(*) FROM ' . $csvTableName
+ . " WHERE first_name = 'yogi'"
+ ));
+ }
+
+ /**
+ * Submit the dataSoure form with the provided form values.
+ *
+ * @param array $sqlFormValues
+ *
+ * @return CRM_Contact_Import_Form_DataSource
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ private function submitDataSourceForm(array $sqlFormValues): CRM_Contact_Import_Form_DataSource {
+ /** @var CRM_Contact_Import_Form_DataSource $form */
+ $form = $this->getFormObject('CRM_Contact_Import_Form_DataSource', $sqlFormValues);
+ $form->buildForm();
+ $form->postProcess();
+ return $form;
}
}
--- /dev/null
+Last Name,email,First Name\r
+Bear ,yogi@yellowstone.park ,Yogi\r
*/
class CRM_Import_DataSource_CsvTest extends CiviUnitTestCase {
+ /**
+ * Prepare for tests.
+ */
+ public function setUp(): void {
+ $this->createLoggedInUser();
+ parent::setUp();
+ }
+
/**
* Test the to csv function.
*
projects / civicrm-core.git / commitdiff