CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This class delegates to the chosen DataSource to grab the data to be imported.
  20  */
  21 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  22
  23   private $_dataSource;
  24
  25   private $_dataSourceIsValid = FALSE;
  26
  27   private $_dataSourceClassFile;
  28
  29   private $_dataSourceClass;
  30
  31   /**
  32    * Set variables up before form is built.
  33    *
  34    * @throws \CRM_Core_Exception
  35    */
  36   public function preProcess() {
  37
  38     //Test database user privilege to create table(Temporary) CRM-4725
  39     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  40     $daoTestPrivilege = new CRM_Core_DAO();
  41     $tempTable1 = CRM_Utils_SQL_TempTable::build()->getName();
  42     $tempTable2 = CRM_Utils_SQL_TempTable::build()->getName();
  43     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable1} (test int) ENGINE=InnoDB");
  44     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable2} (test int) ENGINE=InnoDB");
  45     $daoTestPrivilege->query("DROP TEMPORARY TABLE IF EXISTS {$tempTable1}, {$tempTable2}");
  46     unset($errorScope);
  47
  48     if ($daoTestPrivilege->_lastError) {
  49       $this->invalidConfig(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  50     }
  51
  52     $results = [];
  53     $config = CRM_Core_Config::singleton();
  54     $handler = opendir($config->uploadDir);
  55     $errorFiles = ['sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch'];
  56
  57     // check for post max size avoid when called twice
  58     $snippet = $_GET['snippet'] ?? 0;
  59     if (empty($snippet)) {
  60       CRM_Utils_Number::formatUnitSize(ini_get('post_max_size'), TRUE);
  61     }
  62
  63     while ($file = readdir($handler)) {
  64       if ($file != '.' && $file != '..' &&
  65         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  66       ) {
  67         $results[] = $file;
  68       }
  69     }
  70     closedir($handler);
  71     if (!empty($results)) {
  72       $this->invalidConfig(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', [
  73         1 => implode(', ', $results),
  74         2 => $config->uploadDir,
  75       ]));
  76     }
  77
  78     $this->_dataSourceIsValid = FALSE;
  79     $this->_dataSource = CRM_Utils_Request::retrieveValue(
  80       'dataSource',
  81       'String',
  82       NULL,
  83       FALSE,
  84       'GET'
  85     );
  86
  87     $this->_params = $this->controller->exportValues($this->_name);
  88     if (!$this->_dataSource) {
  89       //considering dataSource as base criteria instead of hidden_dataSource.
  90       $this->_dataSource = CRM_Utils_Array::value('dataSource',
  91         $_POST,
  92         CRM_Utils_Array::value('dataSource',
  93           $this->_params
  94         )
  95       );
  96       $this->assign('showOnlyDataSourceFormPane', FALSE);
  97     }
  98     else {
  99       $this->assign('showOnlyDataSourceFormPane', TRUE);
 100     }
 101
 102     $dataSources = $this->_getDataSources();
 103     if ($this->_dataSource && isset($dataSources[$this->_dataSource])) {
 104       $this->_dataSourceIsValid = TRUE;
 105       $this->assign('showDataSourceFormPane', TRUE);
 106       $dataSourcePath = explode('_', $this->_dataSource);
 107       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 108       $this->assign('dataSourceFormTemplateFile', $templateFile);
 109     }
 110     elseif ($this->_dataSource) {
 111       $this->invalidConfig('Invalid data source');
 112     }
 113   }
 114
 115   /**
 116    * Build the form object.
 117    */
 118   public function buildQuickForm() {
 119
 120     // If there's a dataSource in the query string, we need to load
 121     // the form from the chosen DataSource class
 122     if ($this->_dataSourceIsValid) {
 123       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 124       require_once $this->_dataSourceClassFile;
 125       $this->_dataSourceClass = new $this->_dataSource();
 126       $this->_dataSourceClass->buildQuickForm($this);
 127     }
 128
 129     // Get list of data sources and display them as options
 130     $dataSources = $this->_getDataSources();
 131
 132     $this->assign('urlPath', "civicrm/import");
 133     $this->assign('urlPathVar', 'snippet=4');
 134
 135     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 136       ['onchange' => 'buildDataSourceFormBlock(this.value);']
 137     );
 138
 139     // duplicate handling options
 140     $duplicateOptions = [];
 141     $duplicateOptions[] = $this->createElement('radio',
 142       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 143     );
 144     $duplicateOptions[] = $this->createElement('radio',
 145       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 146     );
 147     $duplicateOptions[] = $this->createElement('radio',
 148       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 149     );
 150     $duplicateOptions[] = $this->createElement('radio',
 151       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 152     );
 153
 154     $this->addGroup($duplicateOptions, 'onDuplicate',
 155       ts('For Duplicate Contacts')
 156     );
 157
 158     $mappingArray = CRM_Core_BAO_Mapping::getMappings('Import Contact');
 159
 160     $this->assign('savedMapping', $mappingArray);
 161     $this->addElement('select', 'savedMapping', ts('Mapping Option'), ['' => ts('- select -')] + $mappingArray);
 162
 163     $js = ['onClick' => "buildSubTypes();buildDedupeRules();"];
 164     // contact types option
 165     $contactOptions = [];
 166     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 167       $contactOptions[] = $this->createElement('radio',
 168         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 169       );
 170     }
 171     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 172       $contactOptions[] = $this->createElement('radio',
 173         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 174       );
 175     }
 176     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 177       $contactOptions[] = $this->createElement('radio',
 178         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 179       );
 180     }
 181
 182     $this->addGroup($contactOptions, 'contactType',
 183       ts('Contact Type')
 184     );
 185
 186     $this->addElement('select', 'subType', ts('Subtype'));
 187     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 188
 189     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 190
 191     $config = CRM_Core_Config::singleton();
 192     $geoCode = FALSE;
 193     if (CRM_Utils_GeocodeProvider::getUsableClassName()) {
 194       $geoCode = TRUE;
 195       $this->addElement('checkbox', 'doGeocodeAddress', ts('Geocode addresses during import?'));
 196     }
 197     $this->assign('geoCode', $geoCode);
 198
 199     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), ['size' => 2]);
 200
 201     if (Civi::settings()->get('address_standardization_provider') == 'USPS') {
 202       $this->addElement('checkbox', 'disableUSPS', ts('Disable USPS address validation during import?'));
 203     }
 204
 205     $this->addButtons([
 206       [
 207         'type' => 'upload',
 208         'name' => ts('Continue'),
 209         'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 210         'isDefault' => TRUE,
 211       ],
 212       [
 213         'type' => 'cancel',
 214         'name' => ts('Cancel'),
 215       ],
 216     ]);
 217   }
 218
 219   /**
 220    * Set the default values of various form elements.
 221    *
 222    * access        public
 223    *
 224    * @return array
 225    *   reference to the array of default values
 226    */
 227   public function setDefaultValues() {
 228     $config = CRM_Core_Config::singleton();
 229     $defaults = [
 230       'dataSource' => 'CRM_Import_DataSource_CSV',
 231       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 232       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 233       'fieldSeparator' => $config->fieldSeparator,
 234     ];
 235
 236     if ($loadeMapping = $this->get('loadedMapping')) {
 237       $this->assign('loadedMapping', $loadeMapping);
 238       $defaults['savedMapping'] = $loadeMapping;
 239     }
 240
 241     return $defaults;
 242   }
 243
 244   /**
 245    * @return array
 246    * @throws Exception
 247    */
 248   private function _getDataSources() {
 249     // Hmm... file-system scanners don't really belong in forms...
 250     if (isset(Civi::$statics[__CLASS__]['datasources'])) {
 251       return Civi::$statics[__CLASS__]['datasources'];
 252     }
 253
 254     // Open the data source dir and scan it for class files
 255     global $civicrm_root;
 256     $dataSourceDir = $civicrm_root . DIRECTORY_SEPARATOR . 'CRM' . DIRECTORY_SEPARATOR . 'Import' . DIRECTORY_SEPARATOR . 'DataSource' . DIRECTORY_SEPARATOR;
 257     $dataSources = [];
 258     if (!is_dir($dataSourceDir)) {
 259       $this->invalidConfig("Import DataSource directory $dataSourceDir does not exist");
 260     }
 261     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 262       $this->invalidConfig("Unable to access DataSource directory $dataSourceDir");
 263     }
 264
 265     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 266       $fileType = filetype($dataSourceDir . $dataSourceFile);
 267       $matches = [];
 268       if (($fileType == 'file' || $fileType == 'link') &&
 269         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 270       ) {
 271         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 272         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 273         $object = new $dataSourceClass();
 274         $info = $object->getInfo();
 275         if ($object->checkPermission()) {
 276           $dataSources[$dataSourceClass] = $info['title'];
 277         }
 278       }
 279     }
 280     closedir($dataSourceHandle);
 281
 282     Civi::$statics[__CLASS__]['datasources'] = $dataSources;
 283     return $dataSources;
 284   }
 285
 286   /**
 287    * Call the DataSource's postProcess method.
 288    */
 289   public function postProcess() {
 290     $this->controller->resetPage('MapField');
 291
 292     if ($this->_dataSourceIsValid) {
 293       // Setup the params array
 294       $this->_params = $this->controller->exportValues($this->_name);
 295
 296       $storeParams = [
 297         'onDuplicate' => $this->exportValue('onDuplicate'),
 298         'dedupe' => $this->exportValue('dedupe'),
 299         'contactType' => $this->exportValue('contactType'),
 300         'contactSubType' => $this->exportValue('subType'),
 301         'dateFormats' => $this->exportValue('dateFormats'),
 302         'savedMapping' => $this->exportValue('savedMapping'),
 303       ];
 304
 305       foreach ($storeParams as $storeName => $value) {
 306         $this->set($storeName, $value);
 307       }
 308       $this->set('disableUSPS', !empty($this->_params['disableUSPS']));
 309
 310       $this->set('dataSource', $this->_params['dataSource']);
 311       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 312
 313       $session = CRM_Core_Session::singleton();
 314       $session->set('dateTypes', $storeParams['dateFormats']);
 315
 316       // Get the PEAR::DB object
 317       $dao = new CRM_Core_DAO();
 318       $db = $dao->getDatabaseConnection();
 319
 320       //hack to prevent multiple tables.
 321       $this->_params['import_table_name'] = $this->get('importTableName');
 322       if (!$this->_params['import_table_name']) {
 323         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 324       }
 325
 326       $this->_dataSourceClass->postProcess($this->_params, $db, $this);
 327
 328       // We should have the data in the DB now, parse it
 329       $importTableName = $this->get('importTableName');
 330       $fieldNames = $this->_prepareImportTable($db, $importTableName);
 331       $mapper = [];
 332
 333       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 334       $parser->setMaxLinesToProcess(100);
 335       $parser->run($importTableName,
 336         $mapper,
 337         CRM_Import_Parser::MODE_MAPFIELD,
 338         $storeParams['contactType'],
 339         $fieldNames['pk'],
 340         $fieldNames['status'],
 341         CRM_Import_Parser::DUPLICATE_SKIP,
 342         NULL, NULL, FALSE,
 343         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 344         $storeParams['contactSubType'],
 345         $storeParams['dedupe']
 346       );
 347
 348       // add all the necessary variables to the form
 349       $parser->set($this);
 350     }
 351     else {
 352       $this->invalidConfig("Invalid DataSource on form post. This shouldn't happen!");
 353     }
 354   }
 355
 356   /**
 357    * Add a PK and status column to the import table so we can track our progress.
 358    * Returns the name of the primary key and status columns
 359    *
 360    * @param $db
 361    * @param string $importTableName
 362    *
 363    * @return array
 364    */
 365   private function _prepareImportTable($db, $importTableName) {
 366     /* TODO: Add a check for an existing _status field;
 367      *  if it exists, create __status instead and return that
 368      */
 369
 370     $statusFieldName = '_status';
 371     $primaryKeyName = '_id';
 372
 373     $this->set('primaryKeyName', $primaryKeyName);
 374     $this->set('statusFieldName', $statusFieldName);
 375
 376     /* Make sure the PK is always last! We rely on this later.
 377      * Should probably stop doing that at some point, but it
 378      * would require moving to associative arrays rather than
 379      * relying on numerical order of the fields. This could in
 380      * turn complicate matters for some DataSources, which
 381      * would also not be good. Decisions, decisions...
 382      */
 383
 384     $alterQuery = "ALTER TABLE $importTableName
 385                        ADD COLUMN $statusFieldName VARCHAR(32)
 386                             DEFAULT 'NEW' NOT NULL,
 387                        ADD COLUMN ${statusFieldName}Msg TEXT,
 388                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 389                                AUTO_INCREMENT";
 390     $db->query($alterQuery);
 391
 392     return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
 393   }
 394
 395   /**
 396    * General function for handling invalid configuration.
 397    *
 398    * I was going to statusBounce them all but when I tested I was 'bouncing' to weird places
 399    * whereas throwing an exception gave no behaviour change. So, I decided to centralise
 400    * and we can 'flip the switch' later.
 401    *
 402    * @param $message
 403    *
 404    * @throws \CRM_Core_Exception
 405    */
 406   protected function invalidConfig($message) {
 407     throw new CRM_Core_Exception($message);
 408   }
 409
 410   /**
 411    * Return a descriptive name for the page, used in wizard header
 412    *
 413    *
 414    * @return string
 415    */
 416   public function getTitle() {
 417     return ts('Choose Data Source');
 418   }
 419
 420 }