CRM/Import/Forms.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 use Civi\Api4\UserJob;
  19
  20 /**
  21  * This class helps the forms within the import flow access submitted & parsed values.
  22  */
  23 class CRM_Import_Forms extends CRM_Core_Form {
  24
  25   /**
  26    * User job id.
  27    *
  28    * This is the primary key of the civicrm_user_job table which is used to
  29    * track the import.
  30    *
  31    * @var int
  32    */
  33   protected $userJobID;
  34
  35   /**
  36    * @return int|null
  37    */
  38   public function getUserJobID(): ?int {
  39     if (!$this->userJobID && $this->get('user_job_id')) {
  40       $this->userJobID = $this->get('user_job_id');
  41     }
  42     return $this->userJobID;
  43   }
  44
  45   /**
  46    * Set user job ID.
  47    *
  48    * @param int $userJobID
  49    */
  50   public function setUserJobID(int $userJobID): void {
  51     $this->userJobID = $userJobID;
  52     // This set allows other forms in the flow ot use $this->get('user_job_id').
  53     $this->set('user_job_id', $userJobID);
  54   }
  55
  56   /**
  57    * User job details.
  58    *
  59    * This is the relevant row from civicrm_user_job.
  60    *
  61    * @var array
  62    */
  63   protected $userJob;
  64
  65   /**
  66    * Get User Job.
  67    *
  68    * API call to retrieve the userJob row.
  69    *
  70    * @return array
  71    *
  72    * @throws \API_Exception
  73    */
  74   protected function getUserJob(): array {
  75     if (!$this->userJob) {
  76       $this->userJob = UserJob::get()
  77         ->addWhere('id', '=', $this->getUserJobID())
  78         ->execute()
  79         ->first();
  80     }
  81     return $this->userJob;
  82   }
  83
  84   /**
  85    * Get submitted values stored in the user job.
  86    *
  87    * @return array
  88    * @throws \API_Exception
  89    */
  90   protected function getUserJobSubmittedValues(): array {
  91     return $this->getUserJob()['metadata']['submitted_values'];
  92   }
  93
  94   /**
  95    * Fields that may be submitted on any form in the flow.
  96    *
  97    * @var string[]
  98    */
  99   protected $submittableFields = [
 100     // Skip column header is actually a field that would be added from the
 101     // datasource - but currently only in contact, it is always there for
 102     // other imports, ditto uploadFile.
 103     'skipColumnHeader' => 'DataSource',
 104     'fieldSeparator' => 'DataSource',
 105     'uploadFile' => 'DataSource',
 106     'contactType' => 'DataSource',
 107     'contactSubType' => 'DataSource',
 108     'dateFormats' => 'DataSource',
 109     'savedMapping' => 'DataSource',
 110     'dataSource' => 'DataSource',
 111     'dedupe_rule_id' => 'DataSource',
 112     'onDuplicate' => 'DataSource',
 113     'disableUSPS' => 'DataSource',
 114     'doGeocodeAddress' => 'DataSource',
 115     // Note we don't add the save mapping instructions for MapField here
 116     // (eg 'updateMapping') - as they really are an action for that form
 117     // rather than part of the mapping config.
 118     'mapper' => 'MapField',
 119   ];
 120
 121   /**
 122    * Get the submitted value, accessing it from whatever form in the flow it is
 123    * submitted on.
 124    *
 125    * @param string $fieldName
 126    *
 127    * @return mixed|null
 128    * @throws \CRM_Core_Exception
 129    */
 130   public function getSubmittedValue(string $fieldName) {
 131     if ($fieldName === 'dataSource') {
 132       // Hard-coded handling for DataSource as it affects the contents of
 133       // getSubmittableFields and can cause a loop.
 134       return $this->controller->exportValue('DataSource', 'dataSource');
 135     }
 136     $mappedValues = $this->getSubmittableFields();
 137     if (array_key_exists($fieldName, $mappedValues)) {
 138       return $this->controller->exportValue($mappedValues[$fieldName], $fieldName);
 139     }
 140     return parent::getSubmittedValue($fieldName);
 141
 142   }
 143
 144   /**
 145    * Get values submitted on any form in the multi-page import flow.
 146    *
 147    * @return array
 148    */
 149   public function getSubmittedValues(): array {
 150     $values = [];
 151     foreach (array_keys($this->getSubmittableFields()) as $key) {
 152       $values[$key] = $this->getSubmittedValue($key);
 153     }
 154     return $values;
 155   }
 156
 157   /**
 158    * Get the available datasource.
 159    *
 160    * Permission dependent, this will look like
 161    * [
 162    *   'CRM_Import_DataSource_CSV' => 'Comma-Separated Values (CSV)',
 163    *   'CRM_Import_DataSource_SQL' => 'SQL Query',
 164    * ]
 165    *
 166    * The label is translated.
 167    *
 168    * @return array
 169    */
 170   protected function getDataSources(): array {
 171     $dataSources = [];
 172     foreach (['CRM_Import_DataSource_SQL', 'CRM_Import_DataSource_CSV'] as $dataSourceClass) {
 173       $object = new $dataSourceClass();
 174       if ($object->checkPermission()) {
 175         $dataSources[$dataSourceClass] = $object->getInfo()['title'];
 176       }
 177     }
 178     return $dataSources;
 179   }
 180
 181   /**
 182    * Get the name of the datasource class.
 183    *
 184    * This function prioritises retrieving from GET and POST over 'submitted'.
 185    * The reason for this is the submitted array will hold the previous submissions
 186    * data until after buildForm is called.
 187    *
 188    * This is problematic in the forward->back flow & option changing flow. As in....
 189    *
 190    * 1) Load DataSource form - initial default datasource is set to CSV and the
 191    * form is via ajax (this calls DataSourceConfig to get the data).
 192    * 2) User changes the source to SQL - the ajax updates the html but the
 193    * form was built with the expectation that the csv-specific fields would be
 194    * required.
 195    * 3) When the user submits Quickform calls preProcess and buildForm and THEN
 196    * retrieves the submitted values based on what has been added in buildForm.
 197    * Only the submitted values for fields added in buildForm are available - but
 198    * these have to be added BEFORE the submitted values are determined. Hence
 199    * we look in the POST or GET to get the updated value.
 200    *
 201    * Note that an imminent refactor will involve storing the values in the
 202    * civicrm_user_job table - this will hopefully help with a known (not new)
 203    * issue whereby the previously submitted values (eg. skipColumnHeader has
 204    * been checked or sql has been filled in) are not loaded via the ajax request.
 205    *
 206    * @return string|null
 207    *
 208    * @throws \CRM_Core_Exception
 209    */
 210   protected function getDataSourceClassName(): string {
 211     $className = CRM_Utils_Request::retrieveValue(
 212       'dataSource',
 213       'String'
 214     );
 215     if (!$className) {
 216       $className = $this->getSubmittedValue('dataSource');
 217     }
 218     if (!$className) {
 219       $className = $this->getDefaultDataSource();
 220     }
 221     if ($this->getDataSources()[$className]) {
 222       return $className;
 223     }
 224     throw new CRM_Core_Exception('Invalid data source');
 225   }
 226
 227   /**
 228    * Allow the datasource class to add fields.
 229    *
 230    * This is called as a snippet in DataSourceConfig and
 231    * also from DataSource::buildForm to add the fields such
 232    * that quick form picks them up.
 233    *
 234    * @throws \CRM_Core_Exception
 235    */
 236   protected function buildDataSourceFields(): void {
 237     $dataSourceClass = $this->getDataSourceObject();
 238     if ($dataSourceClass) {
 239       $dataSourceClass->buildQuickForm($this);
 240     }
 241   }
 242
 243   /**
 244    * Get the relevant datasource object.
 245    *
 246    * @return \CRM_Import_DataSource|null
 247    *
 248    * @throws \CRM_Core_Exception
 249    */
 250   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 251     $className = $this->getDataSourceClassName();
 252     if ($className) {
 253       /* @var CRM_Import_DataSource $dataSource */
 254       return new $className($this->getUserJobID());
 255     }
 256     return NULL;
 257   }
 258
 259   /**
 260    * Allow the datasource class to add fields.
 261    *
 262    * This is called as a snippet in DataSourceConfig and
 263    * also from DataSource::buildForm to add the fields such
 264    * that quick form picks them up.
 265    *
 266    * @throws \CRM_Core_Exception
 267    */
 268   protected function getDataSourceFields(): array {
 269     $className = $this->getDataSourceClassName();
 270     if ($className) {
 271       /* @var CRM_Import_DataSource $dataSourceClass */
 272       $dataSourceClass = new $className();
 273       return $dataSourceClass->getSubmittableFields();
 274     }
 275     return [];
 276   }
 277
 278   /**
 279    * Get the default datasource.
 280    *
 281    * @return string
 282    */
 283   protected function getDefaultDataSource(): string {
 284     return 'CRM_Import_DataSource_CSV';
 285   }
 286
 287   /**
 288    * Get the fields that can be submitted in the Import form flow.
 289    *
 290    * These could be on any form in the flow & are accessed the same way from
 291    * all forms.
 292    *
 293    * @return string[]
 294    * @throws \CRM_Core_Exception
 295    */
 296   protected function getSubmittableFields(): array {
 297     $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource');
 298     return array_merge($this->submittableFields, $dataSourceFields);
 299   }
 300
 301   /**
 302    * Get the contact type selected for the import (on the datasource form).
 303    *
 304    * @return string
 305    *   e.g Individual, Organization, Household.
 306    *
 307    * @throws \CRM_Core_Exception
 308    */
 309   protected function getContactType(): string {
 310     $contactTypeMapping = [
 311       CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 312       CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 313       CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 314     ];
 315     return $contactTypeMapping[$this->getSubmittedValue('contactType')];
 316   }
 317
 318   /**
 319    * Get the contact sub type selected for the import (on the datasource form).
 320    *
 321    * @return string|null
 322    *   e.g Staff.
 323    *
 324    * @throws \CRM_Core_Exception
 325    */
 326   protected function getContactSubType(): ?string {
 327     return $this->getSubmittedValue('contactSubType');
 328   }
 329
 330   /**
 331    * Create a user job to track the import.
 332    *
 333    * @return int
 334    *
 335    * @throws \API_Exception
 336    */
 337   protected function createUserJob(): int {
 338     $id = UserJob::create(FALSE)
 339       ->setValues([
 340         'created_id' => CRM_Core_Session::getLoggedInContactID(),
 341         'type_id:name' => 'contact_import',
 342         'status_id:name' => 'draft',
 343         // This suggests the data could be cleaned up after this.
 344         'expires_date' => '+ 1 week',
 345         'metadata' => [
 346           'submitted_values' => $this->getSubmittedValues(),
 347         ],
 348       ])
 349       ->execute()
 350       ->first()['id'];
 351     $this->setUserJobID($id);
 352     return $id;
 353   }
 354
 355   /**
 356    * @param string $key
 357    * @param array $data
 358    *
 359    * @throws \API_Exception
 360    * @throws \Civi\API\Exception\UnauthorizedException
 361    */
 362   protected function updateUserJobMetadata(string $key, array $data): void {
 363     $metaData = array_merge(
 364       $this->getUserJob()['metadata'],
 365       [$key => $data]
 366     );
 367     UserJob::update(FALSE)
 368       ->addWhere('id', '=', $this->getUserJobID())
 369       ->setValues(['metadata' => $metaData])
 370       ->execute();
 371     $this->userJob['metadata'] = $metaData;
 372   }
 373
 374   /**
 375    * Get column headers for the datasource or empty array if none apply.
 376    *
 377    * This would be the first row of a csv or the fields in an sql query.
 378    *
 379    * If the csv does not have a header row it will be empty.
 380    *
 381    * @return array
 382    *
 383    * @throws \API_Exception
 384    * @throws \CRM_Core_Exception
 385    */
 386   protected function getColumnHeaders(): array {
 387     return $this->getDataSourceObject()->getColumnHeaders();
 388   }
 389
 390   /**
 391    * Get the number of importable columns in the data source.
 392    *
 393    * @return int
 394    *
 395    * @throws \API_Exception
 396    * @throws \CRM_Core_Exception
 397    */
 398   protected function getNumberOfColumns(): int {
 399     return $this->getDataSourceObject()->getNumberOfColumns();
 400   }
 401
 402   /**
 403    * Get x data rows from the datasource.
 404    *
 405    * At this stage we are fetching from what has been stored in the form
 406    * during `postProcess` on the DataSource form.
 407    *
 408    * In the future we will use the dataSource object, likely
 409    * supporting offset as well.
 410    *
 411    * @param int $limit
 412    *
 413    * @return array
 414    *
 415    * @throws \CRM_Core_Exception
 416    * @throws \API_Exception
 417    */
 418   protected function getDataRows(int $limit): array {
 419     return $this->getDataSourceObject()->setLimit($limit)->getRows();
 420   }
 421
 422   /**
 423    * Get the fields available for import selection.
 424    *
 425    * @return array
 426    *   e.g ['first_name' => 'First Name', 'last_name' => 'Last Name'....
 427    *
 428    * @throws \API_Exception
 429    */
 430   protected function getAvailableFields(): array {
 431     $parser = new CRM_Contact_Import_Parser_Contact();
 432     $parser->setUserJobID($this->getUserJobID());
 433     return $parser->getAvailableFields();
 434   }
 435
 436 }