CRM/Import/Forms.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 use Civi\Api4\UserJob;
  19
  20 /**
  21  * This class helps the forms within the import flow access submitted & parsed values.
  22  */
  23 class CRM_Import_Forms extends CRM_Core_Form {
  24
  25   /**
  26    * User job id.
  27    *
  28    * This is the primary key of the civicrm_user_job table which is used to
  29    * track the import.
  30    *
  31    * @var int
  32    */
  33   protected $userJobID;
  34
  35   /**
  36    * @return int|null
  37    */
  38   public function getUserJobID(): ?int {
  39     if (!$this->userJobID && $this->get('user_job_id')) {
  40       $this->userJobID = $this->get('user_job_id');
  41     }
  42     return $this->userJobID;
  43   }
  44
  45   /**
  46    * Set user job ID.
  47    *
  48    * @param int $userJobID
  49    */
  50   public function setUserJobID(int $userJobID): void {
  51     $this->userJobID = $userJobID;
  52     // This set allows other forms in the flow ot use $this->get('user_job_id').
  53     $this->set('user_job_id', $userJobID);
  54   }
  55
  56   /**
  57    * User job details.
  58    *
  59    * This is the relevant row from civicrm_user_job.
  60    *
  61    * @var array
  62    */
  63   protected $userJob;
  64
  65   /**
  66    * Get User Job.
  67    *
  68    * API call to retrieve the userJob row.
  69    *
  70    * @return array
  71    *
  72    * @throws \API_Exception
  73    */
  74   protected function getUserJob(): array {
  75     if (!$this->userJob) {
  76       $this->userJob = UserJob::get()
  77         ->addWhere('id', '=', $this->getUserJobID())
  78         ->execute()
  79         ->first();
  80     }
  81     return $this->userJob;
  82   }
  83
  84   /**
  85    * Get submitted values stored in the user job.
  86    *
  87    * @return array
  88    * @throws \API_Exception
  89    */
  90   protected function getUserJobSubmittedValues(): array {
  91     return $this->getUserJob()['metadata']['submitted_values'];
  92   }
  93
  94   /**
  95    * Fields that may be submitted on any form in the flow.
  96    *
  97    * @var string[]
  98    */
  99   protected $submittableFields = [
 100     // Skip column header is actually a field that would be added from the
 101     // datasource - but currently only in contact, it is always there for
 102     // other imports, ditto uploadFile.
 103     'skipColumnHeader' => 'DataSource',
 104     'fieldSeparator' => 'DataSource',
 105     'uploadFile' => 'DataSource',
 106     'contactType' => 'DataSource',
 107     'contactSubType' => 'DataSource',
 108     'dateFormats' => 'DataSource',
 109     'savedMapping' => 'DataSource',
 110     'dataSource' => 'DataSource',
 111     'dedupe_rule_id' => 'DataSource',
 112     'onDuplicate' => 'DataSource',
 113     'disableUSPS' => 'DataSource',
 114     'doGeocodeAddress' => 'DataSource',
 115     // Note we don't add the save mapping instructions for MapField here
 116     // (eg 'updateMapping') - as they really are an action for that form
 117     // rather than part of the mapping config.
 118     'mapper' => 'MapField',
 119   ];
 120
 121   /**
 122    * Get the submitted value, accessing it from whatever form in the flow it is
 123    * submitted on.
 124    *
 125    * @param string $fieldName
 126    *
 127    * @return mixed|null
 128    * @throws \CRM_Core_Exception
 129    */
 130   public function getSubmittedValue(string $fieldName) {
 131     if ($fieldName === 'dataSource') {
 132       // Hard-coded handling for DataSource as it affects the contents of
 133       // getSubmittableFields and can cause a loop.
 134       return $this->controller->exportValue('DataSource', 'dataSource');
 135     }
 136     $mappedValues = $this->getSubmittableFields();
 137     if (array_key_exists($fieldName, $mappedValues)) {
 138       return $this->controller->exportValue($mappedValues[$fieldName], $fieldName);
 139     }
 140     return parent::getSubmittedValue($fieldName);
 141
 142   }
 143
 144   /**
 145    * Get values submitted on any form in the multi-page import flow.
 146    *
 147    * @return array
 148    */
 149   public function getSubmittedValues(): array {
 150     $values = [];
 151     foreach (array_keys($this->getSubmittableFields()) as $key) {
 152       $values[$key] = $this->getSubmittedValue($key);
 153     }
 154     return $values;
 155   }
 156
 157   /**
 158    * Get the available datasource.
 159    *
 160    * Permission dependent, this will look like
 161    * [
 162    *   'CRM_Import_DataSource_CSV' => 'Comma-Separated Values (CSV)',
 163    *   'CRM_Import_DataSource_SQL' => 'SQL Query',
 164    * ]
 165    *
 166    * The label is translated.
 167    *
 168    * @return array
 169    */
 170   protected function getDataSources(): array {
 171     $dataSources = [];
 172     foreach (['CRM_Import_DataSource_SQL', 'CRM_Import_DataSource_CSV'] as $dataSourceClass) {
 173       $object = new $dataSourceClass();
 174       if ($object->checkPermission()) {
 175         $dataSources[$dataSourceClass] = $object->getInfo()['title'];
 176       }
 177     }
 178     return $dataSources;
 179   }
 180
 181   /**
 182    * Get the name of the datasource class.
 183    *
 184    * This function prioritises retrieving from GET and POST over 'submitted'.
 185    * The reason for this is the submitted array will hold the previous submissions
 186    * data until after buildForm is called.
 187    *
 188    * This is problematic in the forward->back flow & option changing flow. As in....
 189    *
 190    * 1) Load DataSource form - initial default datasource is set to CSV and the
 191    * form is via ajax (this calls DataSourceConfig to get the data).
 192    * 2) User changes the source to SQL - the ajax updates the html but the
 193    * form was built with the expectation that the csv-specific fields would be
 194    * required.
 195    * 3) When the user submits Quickform calls preProcess and buildForm and THEN
 196    * retrieves the submitted values based on what has been added in buildForm.
 197    * Only the submitted values for fields added in buildForm are available - but
 198    * these have to be added BEFORE the submitted values are determined. Hence
 199    * we look in the POST or GET to get the updated value.
 200    *
 201    * Note that an imminent refactor will involve storing the values in the
 202    * civicrm_user_job table - this will hopefully help with a known (not new)
 203    * issue whereby the previously submitted values (eg. skipColumnHeader has
 204    * been checked or sql has been filled in) are not loaded via the ajax request.
 205    *
 206    * @return string|null
 207    *
 208    * @throws \CRM_Core_Exception
 209    */
 210   protected function getDataSourceClassName(): string {
 211     $className = CRM_Utils_Request::retrieveValue(
 212       'dataSource',
 213       'String'
 214     );
 215     if (!$className) {
 216       $className = $this->getSubmittedValue('dataSource');
 217     }
 218     if (!$className) {
 219       $className = $this->getDefaultDataSource();
 220     }
 221     if ($this->getDataSources()[$className]) {
 222       return $className;
 223     }
 224     throw new CRM_Core_Exception('Invalid data source');
 225   }
 226
 227   /**
 228    * Allow the datasource class to add fields.
 229    *
 230    * This is called as a snippet in DataSourceConfig and
 231    * also from DataSource::buildForm to add the fields such
 232    * that quick form picks them up.
 233    *
 234    * @throws \CRM_Core_Exception
 235    */
 236   protected function buildDataSourceFields(): void {
 237     $dataSourceClass = $this->getDataSourceObject();
 238     if ($dataSourceClass) {
 239       $dataSourceClass->buildQuickForm($this);
 240     }
 241   }
 242
 243   /**
 244    * Flush datasource on re-submission of the form.
 245    *
 246    * If the form has been re-submitted the datasource might have changed.
 247    * We tell the dataSource class to remove any tables (and potentially files)
 248    * created last form submission.
 249    *
 250    * If the DataSource in use is unchanged (ie still CSV or still SQL)
 251    * we also pass in the new variables. In theory it could decide that they
 252    * have not actually changed and it doesn't need to do any cleanup.
 253    *
 254    * In practice the datasource classes blast away as they always have for now
 255    * - however, the sql class, for example, might realise the fields it cares
 256    * about are unchanged and not flush the table.
 257    *
 258    * @throws \API_Exception
 259    * @throws \CRM_Core_Exception
 260    */
 261   protected function flushDataSource(): void {
 262     // If the form has been resubmitted the datasource might have changed.
 263     // We give the datasource a chance to clean up any tables it might have
 264     // created. If we are still using the same type of datasource (e.g still
 265     // an sql query
 266     $oldDataSource = $this->getUserJobSubmittedValues()['dataSource'];
 267     $oldDataSourceObject = new $oldDataSource($this->getUserJobID());
 268     $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ? $this->getSubmittedValues() : [];
 269     $oldDataSourceObject->purge($newParams);
 270   }
 271
 272   /**
 273    * Get the relevant datasource object.
 274    *
 275    * @return \CRM_Import_DataSource|null
 276    *
 277    * @throws \CRM_Core_Exception
 278    */
 279   protected function getDataSourceObject(): ?CRM_Import_DataSource {
 280     $className = $this->getDataSourceClassName();
 281     if ($className) {
 282       /* @var CRM_Import_DataSource $dataSource */
 283       return new $className($this->getUserJobID());
 284     }
 285     return NULL;
 286   }
 287
 288   /**
 289    * Allow the datasource class to add fields.
 290    *
 291    * This is called as a snippet in DataSourceConfig and
 292    * also from DataSource::buildForm to add the fields such
 293    * that quick form picks them up.
 294    *
 295    * @throws \CRM_Core_Exception
 296    */
 297   protected function getDataSourceFields(): array {
 298     $className = $this->getDataSourceClassName();
 299     if ($className) {
 300       /* @var CRM_Import_DataSource $dataSourceClass */
 301       $dataSourceClass = new $className();
 302       return $dataSourceClass->getSubmittableFields();
 303     }
 304     return [];
 305   }
 306
 307   /**
 308    * Get the default datasource.
 309    *
 310    * @return string
 311    */
 312   protected function getDefaultDataSource(): string {
 313     return 'CRM_Import_DataSource_CSV';
 314   }
 315
 316   /**
 317    * Get the fields that can be submitted in the Import form flow.
 318    *
 319    * These could be on any form in the flow & are accessed the same way from
 320    * all forms.
 321    *
 322    * @return string[]
 323    * @throws \CRM_Core_Exception
 324    */
 325   protected function getSubmittableFields(): array {
 326     $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource');
 327     return array_merge($this->submittableFields, $dataSourceFields);
 328   }
 329
 330   /**
 331    * Get the contact type selected for the import (on the datasource form).
 332    *
 333    * @return string
 334    *   e.g Individual, Organization, Household.
 335    *
 336    * @throws \CRM_Core_Exception
 337    */
 338   protected function getContactType(): string {
 339     $contactTypeMapping = [
 340       CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual',
 341       CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household',
 342       CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization',
 343     ];
 344     return $contactTypeMapping[$this->getSubmittedValue('contactType')];
 345   }
 346
 347   /**
 348    * Get the contact sub type selected for the import (on the datasource form).
 349    *
 350    * @return string|null
 351    *   e.g Staff.
 352    *
 353    * @throws \CRM_Core_Exception
 354    */
 355   protected function getContactSubType(): ?string {
 356     return $this->getSubmittedValue('contactSubType');
 357   }
 358
 359   /**
 360    * Create a user job to track the import.
 361    *
 362    * @return int
 363    *
 364    * @throws \API_Exception
 365    */
 366   protected function createUserJob(): int {
 367     $id = UserJob::create(FALSE)
 368       ->setValues([
 369         'created_id' => CRM_Core_Session::getLoggedInContactID(),
 370         'type_id:name' => 'contact_import',
 371         'status_id:name' => 'draft',
 372         // This suggests the data could be cleaned up after this.
 373         'expires_date' => '+ 1 week',
 374         'metadata' => [
 375           'submitted_values' => $this->getSubmittedValues(),
 376         ],
 377       ])
 378       ->execute()
 379       ->first()['id'];
 380     $this->setUserJobID($id);
 381     return $id;
 382   }
 383
 384   /**
 385    * @param string $key
 386    * @param array $data
 387    *
 388    * @throws \API_Exception
 389    * @throws \Civi\API\Exception\UnauthorizedException
 390    */
 391   protected function updateUserJobMetadata(string $key, array $data): void {
 392     $metaData = array_merge(
 393       $this->getUserJob()['metadata'],
 394       [$key => $data]
 395     );
 396     UserJob::update(FALSE)
 397       ->addWhere('id', '=', $this->getUserJobID())
 398       ->setValues(['metadata' => $metaData])
 399       ->execute();
 400     $this->userJob['metadata'] = $metaData;
 401   }
 402
 403   /**
 404    * Get column headers for the datasource or empty array if none apply.
 405    *
 406    * This would be the first row of a csv or the fields in an sql query.
 407    *
 408    * If the csv does not have a header row it will be empty.
 409    *
 410    * @return array
 411    *
 412    * @throws \API_Exception
 413    * @throws \CRM_Core_Exception
 414    */
 415   protected function getColumnHeaders(): array {
 416     return $this->getDataSourceObject()->getColumnHeaders();
 417   }
 418
 419   /**
 420    * Get the number of importable columns in the data source.
 421    *
 422    * @return int
 423    *
 424    * @throws \API_Exception
 425    * @throws \CRM_Core_Exception
 426    */
 427   protected function getNumberOfColumns(): int {
 428     return $this->getDataSourceObject()->getNumberOfColumns();
 429   }
 430
 431   /**
 432    * Get x data rows from the datasource.
 433    *
 434    * At this stage we are fetching from what has been stored in the form
 435    * during `postProcess` on the DataSource form.
 436    *
 437    * In the future we will use the dataSource object, likely
 438    * supporting offset as well.
 439    *
 440    * @param int $limit
 441    *
 442    * @return array
 443    *
 444    * @throws \CRM_Core_Exception
 445    * @throws \API_Exception
 446    */
 447   protected function getDataRows(int $limit): array {
 448     return $this->getDataSourceObject()->setLimit($limit)->getRows();
 449   }
 450
 451   /**
 452    * Get the fields available for import selection.
 453    *
 454    * @return array
 455    *   e.g ['first_name' => 'First Name', 'last_name' => 'Last Name'....
 456    *
 457    * @throws \API_Exception
 458    */
 459   protected function getAvailableFields(): array {
 460     $parser = new CRM_Contact_Import_Parser_Contact();
 461     $parser->setUserJobID($this->getUserJobID());
 462     return $parser->getAvailableFields();
 463   }
 464
 465 }