CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  * $Id$
  17  *
  18  */
  19
  20 /**
  21  * This class delegates to the chosen DataSource to grab the data to be imported.
  22  */
  23 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  24
  25   private $_dataSource;
  26
  27   private $_dataSourceIsValid = FALSE;
  28
  29   private $_dataSourceClassFile;
  30
  31   private $_dataSourceClass;
  32
  33   /**
  34    * Set variables up before form is built.
  35    *
  36    * @throws \CRM_Core_Exception
  37    */
  38   public function preProcess() {
  39
  40     //Test database user privilege to create table(Temporary) CRM-4725
  41     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  42     $daoTestPrivilege = new CRM_Core_DAO();
  43     $tempTable1 = CRM_Utils_SQL_TempTable::build()->getName();
  44     $tempTable2 = CRM_Utils_SQL_TempTable::build()->getName();
  45     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable1} (test int) ENGINE=InnoDB");
  46     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable2} (test int) ENGINE=InnoDB");
  47     $daoTestPrivilege->query("DROP TEMPORARY TABLE IF EXISTS {$tempTable1}, {$tempTable2}");
  48     unset($errorScope);
  49
  50     if ($daoTestPrivilege->_lastError) {
  51       $this->invalidConfig(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  52     }
  53
  54     $results = [];
  55     $config = CRM_Core_Config::singleton();
  56     $handler = opendir($config->uploadDir);
  57     $errorFiles = ['sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch'];
  58
  59     // check for post max size avoid when called twice
  60     $snippet = $_GET['snippet'] ?? 0;
  61     if (empty($snippet)) {
  62       CRM_Utils_Number::formatUnitSize(ini_get('post_max_size'), TRUE);
  63     }
  64
  65     while ($file = readdir($handler)) {
  66       if ($file != '.' && $file != '..' &&
  67         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  68       ) {
  69         $results[] = $file;
  70       }
  71     }
  72     closedir($handler);
  73     if (!empty($results)) {
  74       $this->invalidConfig(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', [
  75         1 => implode(', ', $results),
  76         2 => $config->uploadDir,
  77       ]));
  78     }
  79
  80     $this->_dataSourceIsValid = FALSE;
  81     $this->_dataSource = CRM_Utils_Request::retrieveValue(
  82       'dataSource',
  83       'String',
  84       NULL,
  85       FALSE,
  86       'GET'
  87     );
  88
  89     $this->_params = $this->controller->exportValues($this->_name);
  90     if (!$this->_dataSource) {
  91       //considering dataSource as base criteria instead of hidden_dataSource.
  92       $this->_dataSource = CRM_Utils_Array::value('dataSource',
  93         $_POST,
  94         CRM_Utils_Array::value('dataSource',
  95           $this->_params
  96         )
  97       );
  98       $this->assign('showOnlyDataSourceFormPane', FALSE);
  99     }
 100     else {
 101       $this->assign('showOnlyDataSourceFormPane', TRUE);
 102     }
 103
 104     $dataSources = $this->_getDataSources();
 105     if ($this->_dataSource && isset($dataSources[$this->_dataSource])) {
 106       $this->_dataSourceIsValid = TRUE;
 107       $this->assign('showDataSourceFormPane', TRUE);
 108       $dataSourcePath = explode('_', $this->_dataSource);
 109       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 110       $this->assign('dataSourceFormTemplateFile', $templateFile);
 111     }
 112     elseif ($this->_dataSource) {
 113       $this->invalidConfig('Invalid data source');
 114     }
 115   }
 116
 117   /**
 118    * Build the form object.
 119    */
 120   public function buildQuickForm() {
 121
 122     // If there's a dataSource in the query string, we need to load
 123     // the form from the chosen DataSource class
 124     if ($this->_dataSourceIsValid) {
 125       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 126       require_once $this->_dataSourceClassFile;
 127       $this->_dataSourceClass = new $this->_dataSource();
 128       $this->_dataSourceClass->buildQuickForm($this);
 129     }
 130
 131     // Get list of data sources and display them as options
 132     $dataSources = $this->_getDataSources();
 133
 134     $this->assign('urlPath', "civicrm/import");
 135     $this->assign('urlPathVar', 'snippet=4');
 136
 137     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 138       ['onchange' => 'buildDataSourceFormBlock(this.value);']
 139     );
 140
 141     // duplicate handling options
 142     $duplicateOptions = [];
 143     $duplicateOptions[] = $this->createElement('radio',
 144       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 145     );
 146     $duplicateOptions[] = $this->createElement('radio',
 147       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 148     );
 149     $duplicateOptions[] = $this->createElement('radio',
 150       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 151     );
 152     $duplicateOptions[] = $this->createElement('radio',
 153       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 154     );
 155
 156     $this->addGroup($duplicateOptions, 'onDuplicate',
 157       ts('For Duplicate Contacts')
 158     );
 159
 160     $mappingArray = CRM_Core_BAO_Mapping::getMappings('Import Contact');
 161
 162     $this->assign('savedMapping', $mappingArray);
 163     $this->addElement('select', 'savedMapping', ts('Mapping Option'), ['' => ts('- select -')] + $mappingArray);
 164
 165     $js = ['onClick' => "buildSubTypes();buildDedupeRules();"];
 166     // contact types option
 167     $contactOptions = [];
 168     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 169       $contactOptions[] = $this->createElement('radio',
 170         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 171       );
 172     }
 173     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 174       $contactOptions[] = $this->createElement('radio',
 175         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 176       );
 177     }
 178     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 179       $contactOptions[] = $this->createElement('radio',
 180         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 181       );
 182     }
 183
 184     $this->addGroup($contactOptions, 'contactType',
 185       ts('Contact Type')
 186     );
 187
 188     $this->addElement('select', 'subType', ts('Subtype'));
 189     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 190
 191     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 192
 193     $config = CRM_Core_Config::singleton();
 194     $geoCode = FALSE;
 195     if (CRM_Utils_GeocodeProvider::getUsableClassName()) {
 196       $geoCode = TRUE;
 197       $this->addElement('checkbox', 'doGeocodeAddress', ts('Geocode addresses during import?'));
 198     }
 199     $this->assign('geoCode', $geoCode);
 200
 201     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), ['size' => 2]);
 202
 203     if (Civi::settings()->get('address_standardization_provider') == 'USPS') {
 204       $this->addElement('checkbox', 'disableUSPS', ts('Disable USPS address validation during import?'));
 205     }
 206
 207     $this->addButtons([
 208       [
 209         'type' => 'upload',
 210         'name' => ts('Continue'),
 211         'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 212         'isDefault' => TRUE,
 213       ],
 214       [
 215         'type' => 'cancel',
 216         'name' => ts('Cancel'),
 217       ],
 218     ]);
 219   }
 220
 221   /**
 222    * Set the default values of various form elements.
 223    *
 224    * access        public
 225    *
 226    * @return array
 227    *   reference to the array of default values
 228    */
 229   public function setDefaultValues() {
 230     $config = CRM_Core_Config::singleton();
 231     $defaults = [
 232       'dataSource' => 'CRM_Import_DataSource_CSV',
 233       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 234       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 235       'fieldSeparator' => $config->fieldSeparator,
 236     ];
 237
 238     if ($loadeMapping = $this->get('loadedMapping')) {
 239       $this->assign('loadedMapping', $loadeMapping);
 240       $defaults['savedMapping'] = $loadeMapping;
 241     }
 242
 243     return $defaults;
 244   }
 245
 246   /**
 247    * @return array
 248    * @throws Exception
 249    */
 250   private function _getDataSources() {
 251     // Hmm... file-system scanners don't really belong in forms...
 252     if (isset(Civi::$statics[__CLASS__]['datasources'])) {
 253       return Civi::$statics[__CLASS__]['datasources'];
 254     }
 255
 256     // Open the data source dir and scan it for class files
 257     global $civicrm_root;
 258     $dataSourceDir = $civicrm_root . DIRECTORY_SEPARATOR . 'CRM' . DIRECTORY_SEPARATOR . 'Import' . DIRECTORY_SEPARATOR . 'DataSource' . DIRECTORY_SEPARATOR;
 259     $dataSources = [];
 260     if (!is_dir($dataSourceDir)) {
 261       $this->invalidConfig("Import DataSource directory $dataSourceDir does not exist");
 262     }
 263     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 264       $this->invalidConfig("Unable to access DataSource directory $dataSourceDir");
 265     }
 266
 267     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 268       $fileType = filetype($dataSourceDir . $dataSourceFile);
 269       $matches = [];
 270       if (($fileType == 'file' || $fileType == 'link') &&
 271         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 272       ) {
 273         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 274         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 275         $object = new $dataSourceClass();
 276         $info = $object->getInfo();
 277         if ($object->checkPermission()) {
 278           $dataSources[$dataSourceClass] = $info['title'];
 279         }
 280       }
 281     }
 282     closedir($dataSourceHandle);
 283
 284     Civi::$statics[__CLASS__]['datasources'] = $dataSources;
 285     return $dataSources;
 286   }
 287
 288   /**
 289    * Call the DataSource's postProcess method.
 290    */
 291   public function postProcess() {
 292     $this->controller->resetPage('MapField');
 293
 294     if ($this->_dataSourceIsValid) {
 295       // Setup the params array
 296       $this->_params = $this->controller->exportValues($this->_name);
 297
 298       $storeParams = [
 299         'onDuplicate' => $this->exportValue('onDuplicate'),
 300         'dedupe' => $this->exportValue('dedupe'),
 301         'contactType' => $this->exportValue('contactType'),
 302         'contactSubType' => $this->exportValue('subType'),
 303         'dateFormats' => $this->exportValue('dateFormats'),
 304         'savedMapping' => $this->exportValue('savedMapping'),
 305       ];
 306
 307       foreach ($storeParams as $storeName => $value) {
 308         $this->set($storeName, $value);
 309       }
 310       $this->set('disableUSPS', !empty($this->_params['disableUSPS']));
 311
 312       $this->set('dataSource', $this->_params['dataSource']);
 313       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 314
 315       $session = CRM_Core_Session::singleton();
 316       $session->set('dateTypes', $storeParams['dateFormats']);
 317
 318       // Get the PEAR::DB object
 319       $dao = new CRM_Core_DAO();
 320       $db = $dao->getDatabaseConnection();
 321
 322       //hack to prevent multiple tables.
 323       $this->_params['import_table_name'] = $this->get('importTableName');
 324       if (!$this->_params['import_table_name']) {
 325         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 326       }
 327
 328       $this->_dataSourceClass->postProcess($this->_params, $db, $this);
 329
 330       // We should have the data in the DB now, parse it
 331       $importTableName = $this->get('importTableName');
 332       $fieldNames = $this->_prepareImportTable($db, $importTableName);
 333       $mapper = [];
 334
 335       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 336       $parser->setMaxLinesToProcess(100);
 337       $parser->run($importTableName,
 338         $mapper,
 339         CRM_Import_Parser::MODE_MAPFIELD,
 340         $storeParams['contactType'],
 341         $fieldNames['pk'],
 342         $fieldNames['status'],
 343         CRM_Import_Parser::DUPLICATE_SKIP,
 344         NULL, NULL, FALSE,
 345         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 346         $storeParams['contactSubType'],
 347         $storeParams['dedupe']
 348       );
 349
 350       // add all the necessary variables to the form
 351       $parser->set($this);
 352     }
 353     else {
 354       $this->invalidConfig("Invalid DataSource on form post. This shouldn't happen!");
 355     }
 356   }
 357
 358   /**
 359    * Add a PK and status column to the import table so we can track our progress.
 360    * Returns the name of the primary key and status columns
 361    *
 362    * @param $db
 363    * @param string $importTableName
 364    *
 365    * @return array
 366    */
 367   private function _prepareImportTable($db, $importTableName) {
 368     /* TODO: Add a check for an existing _status field;
 369      *  if it exists, create __status instead and return that
 370      */
 371
 372     $statusFieldName = '_status';
 373     $primaryKeyName = '_id';
 374
 375     $this->set('primaryKeyName', $primaryKeyName);
 376     $this->set('statusFieldName', $statusFieldName);
 377
 378     /* Make sure the PK is always last! We rely on this later.
 379      * Should probably stop doing that at some point, but it
 380      * would require moving to associative arrays rather than
 381      * relying on numerical order of the fields. This could in
 382      * turn complicate matters for some DataSources, which
 383      * would also not be good. Decisions, decisions...
 384      */
 385
 386     $alterQuery = "ALTER TABLE $importTableName
 387                        ADD COLUMN $statusFieldName VARCHAR(32)
 388                             DEFAULT 'NEW' NOT NULL,
 389                        ADD COLUMN ${statusFieldName}Msg TEXT,
 390                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 391                                AUTO_INCREMENT";
 392     $db->query($alterQuery);
 393
 394     return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
 395   }
 396
 397   /**
 398    * General function for handling invalid configuration.
 399    *
 400    * I was going to statusBounce them all but when I tested I was 'bouncing' to weird places
 401    * whereas throwing an exception gave no behaviour change. So, I decided to centralise
 402    * and we can 'flip the switch' later.
 403    *
 404    * @param $message
 405    *
 406    * @throws \CRM_Core_Exception
 407    */
 408   protected function invalidConfig($message) {
 409     throw new CRM_Core_Exception($message);
 410   }
 411
 412   /**
 413    * Return a descriptive name for the page, used in wizard header
 414    *
 415    *
 416    * @return string
 417    */
 418   public function getTitle() {
 419     return ts('Choose Data Source');
 420   }
 421
 422 }