3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
18 use Civi\Api4\UserJob
;
19 use League\Csv\Writer
;
22 * This class helps the forms within the import flow access submitted & parsed values.
24 class CRM_Import_Forms
extends CRM_Core_Form
{
29 * This is the primary key of the civicrm_user_job table which is used to
39 public function getUserJobID(): ?
int {
40 if (!$this->userJobID
&& $this->get('user_job_id')) {
41 $this->userJobID
= $this->get('user_job_id');
43 return $this->userJobID
;
49 * @param int $userJobID
51 public function setUserJobID(int $userJobID): void
{
52 $this->userJobID
= $userJobID;
53 // This set allows other forms in the flow ot use $this->get('user_job_id').
54 $this->set('user_job_id', $userJobID);
60 * This is the relevant row from civicrm_user_job.
69 * API call to retrieve the userJob row.
73 * @throws \API_Exception
75 protected function getUserJob(): array {
76 if (!$this->userJob
) {
77 $this->userJob
= UserJob
::get()
78 ->addWhere('id', '=', $this->getUserJobID())
82 return $this->userJob
;
86 * Get submitted values stored in the user job.
89 * @throws \API_Exception
91 protected function getUserJobSubmittedValues(): array {
92 return $this->getUserJob()['metadata']['submitted_values'];
96 * Fields that may be submitted on any form in the flow.
100 protected $submittableFields = [
101 // Skip column header is actually a field that would be added from the
102 // datasource - but currently only in contact, it is always there for
103 // other imports, ditto uploadFile.
104 'skipColumnHeader' => 'DataSource',
105 'fieldSeparator' => 'DataSource',
106 'uploadFile' => 'DataSource',
107 'contactType' => 'DataSource',
108 'contactSubType' => 'DataSource',
109 'dateFormats' => 'DataSource',
110 'savedMapping' => 'DataSource',
111 'dataSource' => 'DataSource',
112 'dedupe_rule_id' => 'DataSource',
113 'onDuplicate' => 'DataSource',
114 'disableUSPS' => 'DataSource',
115 'doGeocodeAddress' => 'DataSource',
116 // Note we don't add the save mapping instructions for MapField here
117 // (eg 'updateMapping') - as they really are an action for that form
118 // rather than part of the mapping config.
119 'mapper' => 'MapField',
123 * Get the submitted value, accessing it from whatever form in the flow it is
126 * @param string $fieldName
129 * @throws \CRM_Core_Exception
131 public function getSubmittedValue(string $fieldName) {
132 if ($fieldName === 'dataSource') {
133 // Hard-coded handling for DataSource as it affects the contents of
134 // getSubmittableFields and can cause a loop.
135 return $this->controller
->exportValue('DataSource', 'dataSource');
137 $mappedValues = $this->getSubmittableFields();
138 if (array_key_exists($fieldName, $mappedValues)) {
139 return $this->controller
->exportValue($mappedValues[$fieldName], $fieldName);
141 return parent
::getSubmittedValue($fieldName);
146 * Get values submitted on any form in the multi-page import flow.
150 public function getSubmittedValues(): array {
152 foreach (array_keys($this->getSubmittableFields()) as $key) {
153 $values[$key] = $this->getSubmittedValue($key);
159 * Get the available datasource.
161 * Permission dependent, this will look like
163 * 'CRM_Import_DataSource_CSV' => 'Comma-Separated Values (CSV)',
164 * 'CRM_Import_DataSource_SQL' => 'SQL Query',
167 * The label is translated.
171 protected function getDataSources(): array {
173 foreach (['CRM_Import_DataSource_SQL', 'CRM_Import_DataSource_CSV'] as $dataSourceClass) {
174 $object = new $dataSourceClass();
175 if ($object->checkPermission()) {
176 $dataSources[$dataSourceClass] = $object->getInfo()['title'];
183 * Get the name of the datasource class.
185 * This function prioritises retrieving from GET and POST over 'submitted'.
186 * The reason for this is the submitted array will hold the previous submissions
187 * data until after buildForm is called.
189 * This is problematic in the forward->back flow & option changing flow. As in....
191 * 1) Load DataSource form - initial default datasource is set to CSV and the
192 * form is via ajax (this calls DataSourceConfig to get the data).
193 * 2) User changes the source to SQL - the ajax updates the html but the
194 * form was built with the expectation that the csv-specific fields would be
196 * 3) When the user submits Quickform calls preProcess and buildForm and THEN
197 * retrieves the submitted values based on what has been added in buildForm.
198 * Only the submitted values for fields added in buildForm are available - but
199 * these have to be added BEFORE the submitted values are determined. Hence
200 * we look in the POST or GET to get the updated value.
202 * Note that an imminent refactor will involve storing the values in the
203 * civicrm_user_job table - this will hopefully help with a known (not new)
204 * issue whereby the previously submitted values (eg. skipColumnHeader has
205 * been checked or sql has been filled in) are not loaded via the ajax request.
207 * @return string|null
209 * @throws \CRM_Core_Exception
211 protected function getDataSourceClassName(): string {
212 $className = CRM_Utils_Request
::retrieveValue(
217 $className = $this->getSubmittedValue('dataSource');
220 $className = $this->getDefaultDataSource();
222 if ($this->getDataSources()[$className]) {
225 throw new CRM_Core_Exception('Invalid data source');
229 * Allow the datasource class to add fields.
231 * This is called as a snippet in DataSourceConfig and
232 * also from DataSource::buildForm to add the fields such
233 * that quick form picks them up.
235 * @throws \CRM_Core_Exception
237 protected function buildDataSourceFields(): void
{
238 $dataSourceClass = $this->getDataSourceObject();
239 if ($dataSourceClass) {
240 $dataSourceClass->buildQuickForm($this);
245 * Flush datasource on re-submission of the form.
247 * If the form has been re-submitted the datasource might have changed.
248 * We tell the dataSource class to remove any tables (and potentially files)
249 * created last form submission.
251 * If the DataSource in use is unchanged (ie still CSV or still SQL)
252 * we also pass in the new variables. In theory it could decide that they
253 * have not actually changed and it doesn't need to do any cleanup.
255 * In practice the datasource classes blast away as they always have for now
256 * - however, the sql class, for example, might realise the fields it cares
257 * about are unchanged and not flush the table.
259 * @throws \API_Exception
260 * @throws \CRM_Core_Exception
262 protected function flushDataSource(): void
{
263 // If the form has been resubmitted the datasource might have changed.
264 // We give the datasource a chance to clean up any tables it might have
265 // created. If we are still using the same type of datasource (e.g still
267 $oldDataSource = $this->getUserJobSubmittedValues()['dataSource'];
268 $oldDataSourceObject = new $oldDataSource($this->getUserJobID());
269 $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ?
$this->getSubmittedValues() : [];
270 $oldDataSourceObject->purge($newParams);
274 * Get the relevant datasource object.
276 * @return \CRM_Import_DataSource|null
278 * @throws \CRM_Core_Exception
280 protected function getDataSourceObject(): ?CRM_Import_DataSource
{
281 $className = $this->getDataSourceClassName();
283 /* @var CRM_Import_DataSource $dataSource */
284 return new $className($this->getUserJobID());
290 * Allow the datasource class to add fields.
292 * This is called as a snippet in DataSourceConfig and
293 * also from DataSource::buildForm to add the fields such
294 * that quick form picks them up.
296 * @throws \CRM_Core_Exception
298 protected function getDataSourceFields(): array {
299 $className = $this->getDataSourceClassName();
301 /* @var CRM_Import_DataSource $dataSourceClass */
302 $dataSourceClass = new $className();
303 return $dataSourceClass->getSubmittableFields();
309 * Get the default datasource.
313 protected function getDefaultDataSource(): string {
314 return 'CRM_Import_DataSource_CSV';
318 * Get the fields that can be submitted in the Import form flow.
320 * These could be on any form in the flow & are accessed the same way from
324 * @throws \CRM_Core_Exception
326 protected function getSubmittableFields(): array {
327 $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource');
328 return array_merge($this->submittableFields
, $dataSourceFields);
332 * Get the contact type selected for the import (on the datasource form).
335 * e.g Individual, Organization, Household.
337 * @throws \CRM_Core_Exception
339 protected function getContactType(): string {
340 $contactTypeMapping = [
341 CRM_Import_Parser
::CONTACT_INDIVIDUAL
=> 'Individual',
342 CRM_Import_Parser
::CONTACT_HOUSEHOLD
=> 'Household',
343 CRM_Import_Parser
::CONTACT_ORGANIZATION
=> 'Organization',
345 return $contactTypeMapping[$this->getSubmittedValue('contactType')];
349 * Get the contact sub type selected for the import (on the datasource form).
351 * @return string|null
354 * @throws \CRM_Core_Exception
356 protected function getContactSubType(): ?
string {
357 return $this->getSubmittedValue('contactSubType');
361 * Create a user job to track the import.
365 * @throws \API_Exception
367 protected function createUserJob(): int {
368 $id = UserJob
::create(FALSE)
370 'created_id' => CRM_Core_Session
::getLoggedInContactID(),
371 'type_id:name' => 'contact_import',
372 'status_id:name' => 'draft',
373 // This suggests the data could be cleaned up after this.
374 'expires_date' => '+ 1 week',
376 'submitted_values' => $this->getSubmittedValues(),
381 $this->setUserJobID($id);
389 * @throws \API_Exception
390 * @throws \Civi\API\Exception\UnauthorizedException
392 protected function updateUserJobMetadata(string $key, array $data): void
{
393 $metaData = array_merge(
394 $this->getUserJob()['metadata'],
397 UserJob
::update(FALSE)
398 ->addWhere('id', '=', $this->getUserJobID())
399 ->setValues(['metadata' => $metaData])
401 $this->userJob
['metadata'] = $metaData;
405 * Get column headers for the datasource or empty array if none apply.
407 * This would be the first row of a csv or the fields in an sql query.
409 * If the csv does not have a header row it will be empty.
413 * @throws \API_Exception
414 * @throws \CRM_Core_Exception
416 protected function getColumnHeaders(): array {
417 return $this->getDataSourceObject()->getColumnHeaders();
421 * Get the number of importable columns in the data source.
425 * @throws \API_Exception
426 * @throws \CRM_Core_Exception
428 protected function getNumberOfColumns(): int {
429 return $this->getDataSourceObject()->getNumberOfColumns();
433 * Get x data rows from the datasource.
435 * At this stage we are fetching from what has been stored in the form
436 * during `postProcess` on the DataSource form.
438 * In the future we will use the dataSource object, likely
439 * supporting offset as well.
442 * One or more of the statues available - e.g
443 * CRM_Import_Parser::VALID
444 * or [CRM_Import_Parser::ERROR, CRM_Import_Parser::VALID]
446 * @throws \CRM_Core_Exception
447 * @throws \API_Exception
449 protected function getDataRows($statuses = [], int $limit = 0): array {
450 $statuses = (array) $statuses;
451 return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows();
455 * Get the number of rows with the specified status.
457 * @param array|int $statuses
461 * @throws \API_Exception
462 * @throws \CRM_Core_Exception
464 protected function getRowCount($statuses = []) {
465 $statuses = (array) $statuses;
466 return $this->getDataSourceObject()->getRowCount($statuses);
470 * Outputs and downloads the csv of outcomes from an import job.
472 * This gets the rows from the temp table that match the relevant status
473 * and output them as a csv.
475 * @throws \API_Exception
476 * @throws \League\Csv\CannotInsertRecord
477 * @throws \CRM_Core_Exception
479 public static function outputCSV(): void
{
480 $userJobID = CRM_Utils_Request
::retrieveValue('user_job_id', 'Integer', NULL, TRUE);
481 $status = CRM_Utils_Request
::retrieveValue('status', 'String', NULL, TRUE);
482 $saveFileName = CRM_Import_Parser
::saveFileName($status);
484 $form = new CRM_Import_Forms();
485 $form->controller
= new CRM_Core_Controller();
486 $form->set('user_job_id', $userJobID);
489 $writer = Writer
::createFromFileObject(new SplTempFileObject());
490 $headers = $form->getColumnHeaders();
492 array_unshift($headers, ts('Reason'));
493 array_unshift($headers, ts('Line Number'));
494 $writer->insertOne($headers);
496 $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']);
497 // Note this might be more inefficient that iterating the result
498 // set & doing insertOne - possibly something to explore later.
499 $writer->insertAll($form->getDataRows($status));
501 CRM_Utils_System
::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
502 CRM_Utils_System
::setHttpHeader('Content-Description', 'File Transfer');
503 CRM_Utils_System
::setHttpHeader('Content-Type', 'text/csv; charset=UTF-8');
504 $writer->output($saveFileName);
505 CRM_Utils_System
::civiExit();
509 * When outputting the row as a csv, more the last 2 rows to the start.
511 * This is because the id and status message fields are at the end. It may make sense
512 * to move them to the start later, when order code cleanup has happened...
514 * @param array $record
516 public static function reorderOutput(array $record): array {
517 $rowNumber = array_pop($record);
518 $message = array_pop($record);
519 // Also pop off the status - but we are not going to use this at this stage.
525 array_unshift($record, $message);
526 array_unshift($record, $rowNumber);
531 * Get the url to download the relevant csv file.
532 * @param string $status
536 protected function getDownloadURL(string $status): string {
537 return CRM_Utils_System
::url('civicrm/import/outcome', [
538 'user_job_id' => $this->get('user_job_id'),
545 * Get the fields available for import selection.
548 * e.g ['first_name' => 'First Name', 'last_name' => 'Last Name'....
550 * @throws \API_Exception
552 protected function getAvailableFields(): array {
553 $parser = new CRM_Contact_Import_Parser_Contact();
554 $parser->setUserJobID($this->getUserJobID());
555 return $parser->getAvailableFields();