CRM/Import/Forms.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 use Civi\Api4\UserJob;
  19 use League\Csv\Writer;
  20
  21 /**
  22  * This class helps the forms within the import flow access submitted & parsed values.
  23  */
  24 class CRM_Import_Forms extends CRM_Core_Form {
  25
  26   /**
  27    * User job id.
  28    *
  29    * This is the primary key of the civicrm_user_job table which is used to
  30    * track the import.
  31    *
  32    * @var int
  33    */
  34   protected $userJobID;
  35
  36   /**
  37    * @return int|null
  38    */
  39   public function getUserJobID(): ?int {
  40     if (!$this->userJobID && $this->get('user_job_id')) {
  41       $this->userJobID = $this->get('user_job_id');
  42     }
  43     return $this->userJobID;
  44   }
  45
  46   /**
  47    * Set user job ID.
  48    *
  49    * @param int $userJobID
  50    */
  51   public function setUserJobID(int $userJobID): void {
  52     $this->userJobID = $userJobID;
  53     // This set allows other forms in the flow ot use $this->get('user_job_id').
  54     $this->set('user_job_id', $userJobID);
  55   }
  56
  57   /**
  58    * User job details.
  59    *
  60    * This is the relevant row from civicrm_user_job.
  61    *
  62    * @var array
  63    */
  64   protected $userJob;
  65
  66   /**
  67    * @var \CRM_Import_Parser
  68    */
  69   protected $parser;
  70
  71   /**
  72    * Get User Job.
  73    *
  74    * API call to retrieve the userJob row.
  75    *
  76    * @return array
  77    *
  78    * @throws \API_Exception
  79    */
  80   protected function getUserJob(): array {
  81     if (!$this->userJob) {
  82       $this->userJob = UserJob::get()
  83         ->addWhere('id', '=', $this->getUserJobID())
  84         ->execute()
  85         ->first();
  86     }
  87     return $this->userJob;
  88   }
  89
  90   /**
  91    * Get submitted values stored in the user job.
  92    *
  93    * @return array
  94    * @throws \API_Exception
  95    */
  96   protected function getUserJobSubmittedValues(): array {
  97     return $this->getUserJob()['metadata']['submitted_values'];
  98   }
  99
 100   /**
 101    * Fields that may be submitted on any form in the flow.
 102    *
 103    * @var string[]
 104    */
 105   protected $submittableFields = [
 106     // Skip column header is actually a field that would be added from the
 107     // datasource - but currently only in contact, it is always there for
 108     // other imports, ditto uploadFile.
 109     'skipColumnHeader' => 'DataSource',
 110     'fieldSeparator' => 'DataSource',
 111     'uploadFile' => 'DataSource',
 112     'contactType' => 'DataSource',
 113     'contactSubType' => 'DataSource',
 114     'dateFormats' => 'DataSource',
 115     'savedMapping' => 'DataSource',
 116     'dataSource' => 'DataSource',
 117     'dedupe_rule_id' => 'DataSource',
 118     'onDuplicate' => 'DataSource',
 119     'disableUSPS' => 'DataSource',
 120     'doGeocodeAddress' => 'DataSource',
 121     // Note we don't add the save mapping instructions for MapField here
 122     // (eg 'updateMapping') - as they really are an action for that form
 123     // rather than part of the mapping config.
 124     'mapper' => 'MapField',
 125   ];
 126
 127   /**
 128    * Get the submitted value, accessing it from whatever form in the flow it is
 129    * submitted on.
 130    *
 131    * @param string $fieldName
 132    *
 133    * @return mixed|null
 134    * @throws \CRM_Core_Exception
 135    */
 136   public function getSubmittedValue(string $fieldName) {
 137     if ($fieldName === 'dataSource') {
 138       // Hard-coded handling for DataSource as it affects the contents of
 139       // getSubmittableFields and can cause a loop.
 140       return $this->controller->exportValue('DataSource', 'dataSource');
 141     }
 142     $mappedValues = $this->getSubmittableFields();
 143     if (array_key_exists($fieldName, $mappedValues)) {
 144       return $this->controller->exportValue($mappedValues[$fieldName], $fieldName);
 145     }
 146     return parent::getSubmittedValue($fieldName);
 147
 148   }
 149
 150   /**
 151    * Get values submitted on any form in the multi-page import flow.
 152    *
 153    * @return array
 154    */
 155   public function getSubmittedValues(): array {
 156     $values = [];
 157     foreach (array_keys($this->getSubmittableFields()) as $key) {
 158       $values[$key] = $this->getSubmittedValue($key);
 159     }
 160     return $values;
 161   }
 162
 163   /**
 164    * Get the available datasource.
 165    *
 166    * Permission dependent, this will look like
 167    * [
 168    *   'CRM_Import_DataSource_CSV' => 'Comma-Separated Values (CSV)',
 169    *   'CRM_Import_DataSource_SQL' => 'SQL Query',
 170    * ]
 171    *
 172    * The label is translated.
 173    *
 174    * @return array
 175    */
 176   protected function getDataSources(): array {
 177     $dataSources = [];
 178     foreach (['CRM_Import_DataSource_SQL', 'CRM_Import_DataSource_CSV'] as $dataSourceClass) {
 179       $object = new $dataSourceClass();
 180       if ($object->checkPermission()) {
 181         $dataSources[$dataSourceClass] = $object->getInfo()['title'];
 182       }
 183     }
 184     return $dataSources;
 185   }
 186
 187   /**
 188    * Get the name of the datasource class.
 189    *
 190    * This function prioritises retrieving from GET and POST over 'submitted'.
 191    * The reason for this is the submitted array will hold the previous submissions
 192    * data until after buildForm is called.
 193    *
 194    * This is problematic in the forward->back flow & option changing flow. As in....
 195    *
 196    * 1) Load DataSource form - initial default datasource is set to CSV and the
 197    * form is via ajax (this calls DataSourceConfig to get the data).
 198    * 2) User changes the source to SQL - the ajax updates the html but the
 199    * form was built with the expectation that the csv-specific fields would be
 200    * required.
 201    * 3) When the user submits Quickform calls preProcess and buildForm and THEN
 202    * retrieves the submitted values based on what has been added in buildForm.
 203    * Only the submitted values for fields added in buildForm are available - but
 204    * these have to be added BEFORE the submitted values are determined. Hence
 205    * we look in the POST or GET to get the updated value.
 206    *
 207    * Note that an imminent refactor will involve storing the values in the
 208    * civicrm_user_job table - this will hopefully help with a known (not new)
 209    * issue whereby the previously submitted values (eg. skipColumnHeader has
 210    * been checked or sql has been filled in) are not loaded via the ajax request.
 211    *
 212    * @return string|null
 213    *
 214    * @throws \CRM_Core_Exception
 215    */
 216   protected function getDataSourceClassName(): string {
 217     $className = CRM_Utils_Request::retrieveValue(
 218       'dataSource',
 219       'String'
 220     );
 221     if (!$className) {
 222       $className = $this->getSubmittedValue('dataSource');
 223     }
 224     if (!$className) {
 225       $className = $this->getDefaultDataSource();
 226     }
 227     if ($this->getDataSources()[$className]) {
 228       return $className;
 229     }
 230     throw new CRM_Core_Exception('Invalid data source');
 231   }
 232
 233   /**
 234    * Allow the datasource class to add fields.
 235    *
 236    * This is called as a snippet in DataSourceConfig and
 237    * also from DataSource::buildForm to add the fields such
 238    * that quick form picks them up.
 239    *
 240    * @throws \CRM_Core_Exception
 241    */
 242   protected function buildDataSourceFields(): void {
 243     $dataSourceClass = $this->getDataSourceObject();
 244     if ($dataSourceClass) {
 245       $dataSourceClass->buildQuickForm($this);
 246     }
 247   }
 248
 249   /**
 250    * Flush datasource on re-submission of the form.
 251    *
 252    * If the form has been re-submitted the datasource might have changed.
 253    * We tell the dataSource class to remove any tables (and potentially files)
 254    * created last form submission.
 255    *
 256    * If the DataSource in use is unchanged (ie still CSV or still SQL)
 257    * we also pass in the new variables. In theory it could decide that they
 258    * have not actually changed and it doesn't need to do any cleanup.
 259    *
 260    * In practice the datasource classes blast away as they always have for now
 261    * - however, the sql class, for example, might realise the fields it cares
 262    * about are unchanged and not flush the table.
 263    *
 264    * @throws \API_Exception
 265    * @throws \CRM_Core_Exception
 266    */
 267   protected function flushDataSource(): void {
 268     // If the form has been resubmitted the datasource might have changed.
 269     // We give the datasource a chance to clean up any tables it might have
 270     // created. If we are still using the same type of datasource (e.g still
 271     // an sql query
 272     $oldDataSource = $this->getUserJobSubmittedValues()['dataSource'];
 273     $oldDataSourceObject = new $oldDataSource($this->getUserJobID());
 274     $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ? $this->getSubmittedValues() : [];
 275     $oldDataSourceObject->purge($newParams);
 276   }
 277
 278   /**
 279    * Get the relevant datasource object.
 280    *
 281    * @return \CRM_Import_DataSource|null
 282    *
 283    * @throws \CRM_Core_Exception
 284    */
 285   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 286     $className = $this->getDataSourceClassName();
 287     if ($className) {
 288       /* @var CRM_Import_DataSource $dataSource */
 289       return new $className($this->getUserJobID());
 290     }
 291     return NULL;
 292   }
 293
 294   /**
 295    * Allow the datasource class to add fields.
 296    *
 297    * This is called as a snippet in DataSourceConfig and
 298    * also from DataSource::buildForm to add the fields such
 299    * that quick form picks them up.
 300    *
 301    * @throws \CRM_Core_Exception
 302    */
 303   protected function getDataSourceFields(): array {
 304     $className = $this->getDataSourceClassName();
 305     if ($className) {
 306       /* @var CRM_Import_DataSource $dataSourceClass */
 307       $dataSourceClass = new $className();
 308       return $dataSourceClass->getSubmittableFields();
 309     }
 310     return [];
 311   }
 312
 313   /**
 314    * Get the default datasource.
 315    *
 316    * @return string
 317    */
 318   protected function getDefaultDataSource(): string {
 319     return 'CRM_Import_DataSource_CSV';
 320   }
 321
 322   /**
 323    * Get the fields that can be submitted in the Import form flow.
 324    *
 325    * These could be on any form in the flow & are accessed the same way from
 326    * all forms.
 327    *
 328    * @return string[]
 329    * @throws \CRM_Core_Exception
 330    */
 331   protected function getSubmittableFields(): array {
 332     $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource');
 333     return array_merge($this->submittableFields, $dataSourceFields);
 334   }
 335
 336   /**
 337    * Get the contact type selected for the import (on the datasource form).
 338    *
 339    * @return string
 340    *   e.g Individual, Organization, Household.
 341    *
 342    * @throws \CRM_Core_Exception
 343    */
 344   protected function getContactType(): string {
 345     $contactTypeMapping = [
 346       CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 347       CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 348       CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 349     ];
 350     return $contactTypeMapping[$this->getSubmittedValue('contactType')];
 351   }
 352
 353   /**
 354    * Get the contact sub type selected for the import (on the datasource form).
 355    *
 356    * @return string|null
 357    *   e.g Staff.
 358    *
 359    * @throws \CRM_Core_Exception
 360    */
 361   protected function getContactSubType(): ?string {
 362     return $this->getSubmittedValue('contactSubType');
 363   }
 364
 365   /**
 366    * Create a user job to track the import.
 367    *
 368    * @return int
 369    *
 370    * @throws \API_Exception
 371    */
 372   protected function createUserJob(): int {
 373     $id = UserJob::create(FALSE)
 374       ->setValues([
 375         'created_id' => CRM_Core_Session::getLoggedInContactID(),
 376         'type_id:name' => 'contact_import',
 377         'status_id:name' => 'draft',
 378         // This suggests the data could be cleaned up after this.
 379         'expires_date' => '+ 1 week',
 380         'metadata' => [
 381           'submitted_values' => $this->getSubmittedValues(),
 382         ],
 383       ])
 384       ->execute()
 385       ->first()['id'];
 386     $this->setUserJobID($id);
 387     return $id;
 388   }
 389
 390   /**
 391    * @param string $key
 392    * @param array $data
 393    *
 394    * @throws \API_Exception
 395    * @throws \Civi\API\Exception\UnauthorizedException
 396    */
 397   protected function updateUserJobMetadata(string $key, array $data): void {
 398     $metaData = array_merge(
 399       $this->getUserJob()['metadata'],
 400       [$key => $data]
 401     );
 402     UserJob::update(FALSE)
 403       ->addWhere('id', '=', $this->getUserJobID())
 404       ->setValues(['metadata' => $metaData])
 405       ->execute();
 406     $this->userJob['metadata'] = $metaData;
 407   }
 408
 409   /**
 410    * Get column headers for the datasource or empty array if none apply.
 411    *
 412    * This would be the first row of a csv or the fields in an sql query.
 413    *
 414    * If the csv does not have a header row it will be empty.
 415    *
 416    * @return array
 417    *
 418    * @throws \API_Exception
 419    * @throws \CRM_Core_Exception
 420    */
 421   protected function getColumnHeaders(): array {
 422     return $this->getDataSourceObject()->getColumnHeaders();
 423   }
 424
 425   /**
 426    * Get the number of importable columns in the data source.
 427    *
 428    * @return int
 429    *
 430    * @throws \API_Exception
 431    * @throws \CRM_Core_Exception
 432    */
 433   protected function getNumberOfColumns(): int {
 434     return $this->getDataSourceObject()->getNumberOfColumns();
 435   }
 436
 437   /**
 438    * Get x data rows from the datasource.
 439    *
 440    * At this stage we are fetching from what has been stored in the form
 441    * during `postProcess` on the DataSource form.
 442    *
 443    * In the future we will use the dataSource object, likely
 444    * supporting offset as well.
 445    *
 446    * @return array|int
 447    *   One or more of the statues available - e.g
 448    *   CRM_Import_Parser::VALID
 449    *   or [CRM_Import_Parser::ERROR, CRM_Import_Parser::VALID]
 450    *
 451    * @throws \CRM_Core_Exception
 452    * @throws \API_Exception
 453    */
 454   protected function getDataRows($statuses = [], int $limit = 0): array {
 455     $statuses = (array) $statuses;
 456     return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows();
 457   }
 458
 459   /**
 460    * Get the number of rows with the specified status.
 461    *
 462    * @param array|int $statuses
 463    *
 464    * @return int
 465    *
 466    * @throws \API_Exception
 467    * @throws \CRM_Core_Exception
 468    */
 469   protected function getRowCount($statuses = []) {
 470     $statuses = (array) $statuses;
 471     return $this->getDataSourceObject()->getRowCount($statuses);
 472   }
 473
 474   /**
 475    * Outputs and downloads the csv of outcomes from an import job.
 476    *
 477    * This gets the rows from the temp table that match the relevant status
 478    * and output them as a csv.
 479    *
 480    * @throws \API_Exception
 481    * @throws \League\Csv\CannotInsertRecord
 482    * @throws \CRM_Core_Exception
 483    */
 484   public static function outputCSV(): void {
 485     $userJobID = CRM_Utils_Request::retrieveValue('user_job_id', 'Integer', NULL, TRUE);
 486     $status = CRM_Utils_Request::retrieveValue('status', 'String', NULL, TRUE);
 487     $saveFileName = CRM_Import_Parser::saveFileName($status);
 488
 489     $form = new CRM_Import_Forms();
 490     $form->controller = new CRM_Core_Controller();
 491     $form->set('user_job_id', $userJobID);
 492
 493     $form->getUserJob();
 494     $writer = Writer::createFromFileObject(new SplTempFileObject());
 495     $headers = $form->getColumnHeaders();
 496     if ($headers) {
 497       array_unshift($headers, ts('Reason'));
 498       array_unshift($headers, ts('Line Number'));
 499       $writer->insertOne($headers);
 500     }
 501     $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']);
 502     // Note this might be more inefficient that iterating the result
 503     // set & doing insertOne - possibly something to explore later.
 504     $writer->insertAll($form->getDataRows($status));
 505
 506     CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
 507     CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer');
 508     CRM_Utils_System::setHttpHeader('Content-Type', 'text/csv; charset=UTF-8');
 509     $writer->output($saveFileName);
 510     CRM_Utils_System::civiExit();
 511   }
 512
 513   /**
 514    * When outputting the row as a csv, more the last 2 rows to the start.
 515    *
 516    * This is because the id and status message fields are at the end. It may make sense
 517    * to move them to the start later, when order code cleanup has happened...
 518    *
 519    * @param array $record
 520    */
 521   public static function reorderOutput(array $record): array {
 522     $rowNumber = array_pop($record);
 523     $message = array_pop($record);
 524     // Also pop off the status - but we are not going to use this at this stage.
 525     array_pop($record);
 526     // Related entities
 527     array_pop($record);
 528     // Entity_id
 529     array_pop($record);
 530     array_unshift($record, $message);
 531     array_unshift($record, $rowNumber);
 532     return $record;
 533   }
 534
 535   /**
 536    * Get the url to download the relevant csv file.
 537    * @param string $status
 538    *
 539    * @return string
 540    */
 541   protected function getDownloadURL(string $status): string {
 542     return CRM_Utils_System::url('civicrm/import/outcome', [
 543       'user_job_id' => $this->get('user_job_id'),
 544       'status' => $status,
 545       'reset' => 1,
 546     ]);
 547   }
 548
 549   /**
 550    * Get the fields available for import selection.
 551    *
 552    * @return array
 553    *   e.g ['first_name' => 'First Name', 'last_name' => 'Last Name'....
 554    *
 555    * @throws \API_Exception
 556    */
 557   protected function getAvailableFields(): array {
 558     return $this->getParser()->getAvailableFields();
 559   }
 560
 561   /**
 562    * Get an instance of the parser class.
 563    *
 564    * @return \CRM_Contact_Import_Parser_Contact|\CRM_Contribute_Import_Parser_Contribution
 565    */
 566   protected function getParser() {
 567     return NULL;
 568   }
 569
 570 }