CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2020                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2009 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2020
  32  */
  33
  34 /**
  35  * This class delegates to the chosen DataSource to grab the data to be imported.
  36  */
  37 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  38
  39   private $_dataSource;
  40
  41   private $_dataSourceIsValid = FALSE;
  42
  43   private $_dataSourceClassFile;
  44
  45   private $_dataSourceClass;
  46
  47   /**
  48    * Set variables up before form is built.
  49    *
  50    * @throws \CRM_Core_Exception
  51    */
  52   public function preProcess() {
  53
  54     //Test database user privilege to create table(Temporary) CRM-4725
  55     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  56     $daoTestPrivilege = new CRM_Core_DAO();
  57     $tempTable1 = CRM_Utils_SQL_TempTable::build()->getName();
  58     $tempTable2 = CRM_Utils_SQL_TempTable::build()->getName();
  59     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable1} (test int) ENGINE=InnoDB");
  60     $daoTestPrivilege->query("CREATE TEMPORARY TABLE {$tempTable2} (test int) ENGINE=InnoDB");
  61     $daoTestPrivilege->query("DROP TEMPORARY TABLE IF EXISTS {$tempTable1}, {$tempTable2}");
  62     unset($errorScope);
  63
  64     if ($daoTestPrivilege->_lastError) {
  65       $this->invalidConfig(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  66     }
  67
  68     $results = [];
  69     $config = CRM_Core_Config::singleton();
  70     $handler = opendir($config->uploadDir);
  71     $errorFiles = ['sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch'];
  72
  73     // check for post max size avoid when called twice
  74     $snippet = CRM_Utils_Array::value('snippet', $_GET, 0);
  75     if (empty($snippet)) {
  76       CRM_Utils_Number::formatUnitSize(ini_get('post_max_size'), TRUE);
  77     }
  78
  79     while ($file = readdir($handler)) {
  80       if ($file != '.' && $file != '..' &&
  81         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  82       ) {
  83         $results[] = $file;
  84       }
  85     }
  86     closedir($handler);
  87     if (!empty($results)) {
  88       $this->invalidConfig(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', [
  89         1 => implode(', ', $results),
  90         2 => $config->uploadDir,
  91       ]));
  92     }
  93
  94     $this->_dataSourceIsValid = FALSE;
  95     $this->_dataSource = CRM_Utils_Request::retrieveValue(
  96       'dataSource',
  97       'String',
  98       NULL,
  99       FALSE,
 100       'GET'
 101     );
 102
 103     $this->_params = $this->controller->exportValues($this->_name);
 104     if (!$this->_dataSource) {
 105       //considering dataSource as base criteria instead of hidden_dataSource.
 106       $this->_dataSource = CRM_Utils_Array::value('dataSource',
 107         $_POST,
 108         CRM_Utils_Array::value('dataSource',
 109           $this->_params
 110         )
 111       );
 112       $this->assign('showOnlyDataSourceFormPane', FALSE);
 113     }
 114     else {
 115       $this->assign('showOnlyDataSourceFormPane', TRUE);
 116     }
 117
 118     $dataSources = $this->_getDataSources();
 119     if ($this->_dataSource && isset($dataSources[$this->_dataSource])) {
 120       $this->_dataSourceIsValid = TRUE;
 121       $this->assign('showDataSourceFormPane', TRUE);
 122       $dataSourcePath = explode('_', $this->_dataSource);
 123       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 124       $this->assign('dataSourceFormTemplateFile', $templateFile);
 125     }
 126     elseif ($this->_dataSource) {
 127       $this->invalidConfig('Invalid data source');
 128     }
 129   }
 130
 131   /**
 132    * Build the form object.
 133    */
 134   public function buildQuickForm() {
 135
 136     // If there's a dataSource in the query string, we need to load
 137     // the form from the chosen DataSource class
 138     if ($this->_dataSourceIsValid) {
 139       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 140       require_once $this->_dataSourceClassFile;
 141       $this->_dataSourceClass = new $this->_dataSource();
 142       $this->_dataSourceClass->buildQuickForm($this);
 143     }
 144
 145     // Get list of data sources and display them as options
 146     $dataSources = $this->_getDataSources();
 147
 148     $this->assign('urlPath', "civicrm/import");
 149     $this->assign('urlPathVar', 'snippet=4');
 150
 151     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 152       ['onchange' => 'buildDataSourceFormBlock(this.value);']
 153     );
 154
 155     // duplicate handling options
 156     $duplicateOptions = [];
 157     $duplicateOptions[] = $this->createElement('radio',
 158       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 159     );
 160     $duplicateOptions[] = $this->createElement('radio',
 161       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 162     );
 163     $duplicateOptions[] = $this->createElement('radio',
 164       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 165     );
 166     $duplicateOptions[] = $this->createElement('radio',
 167       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 168     );
 169
 170     $this->addGroup($duplicateOptions, 'onDuplicate',
 171       ts('For Duplicate Contacts')
 172     );
 173
 174     $mappingArray = CRM_Core_BAO_Mapping::getMappings('Import Contact');
 175
 176     $this->assign('savedMapping', $mappingArray);
 177     $this->addElement('select', 'savedMapping', ts('Mapping Option'), ['' => ts('- select -')] + $mappingArray);
 178
 179     $js = ['onClick' => "buildSubTypes();buildDedupeRules();"];
 180     // contact types option
 181     $contactOptions = [];
 182     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 183       $contactOptions[] = $this->createElement('radio',
 184         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 185       );
 186     }
 187     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 188       $contactOptions[] = $this->createElement('radio',
 189         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 190       );
 191     }
 192     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 193       $contactOptions[] = $this->createElement('radio',
 194         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 195       );
 196     }
 197
 198     $this->addGroup($contactOptions, 'contactType',
 199       ts('Contact Type')
 200     );
 201
 202     $this->addElement('select', 'subType', ts('Subtype'));
 203     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 204
 205     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 206
 207     $config = CRM_Core_Config::singleton();
 208     $geoCode = FALSE;
 209     if (CRM_Utils_GeocodeProvider::getUsableClassName()) {
 210       $geoCode = TRUE;
 211       $this->addElement('checkbox', 'doGeocodeAddress', ts('Geocode addresses during import?'));
 212     }
 213     $this->assign('geoCode', $geoCode);
 214
 215     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), ['size' => 2]);
 216
 217     if (Civi::settings()->get('address_standardization_provider') == 'USPS') {
 218       $this->addElement('checkbox', 'disableUSPS', ts('Disable USPS address validation during import?'));
 219     }
 220
 221     $this->addButtons([
 222       [
 223         'type' => 'upload',
 224         'name' => ts('Continue'),
 225         'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 226         'isDefault' => TRUE,
 227       ],
 228       [
 229         'type' => 'cancel',
 230         'name' => ts('Cancel'),
 231       ],
 232     ]);
 233   }
 234
 235   /**
 236    * Set the default values of various form elements.
 237    *
 238    * access        public
 239    *
 240    * @return array
 241    *   reference to the array of default values
 242    */
 243   public function setDefaultValues() {
 244     $config = CRM_Core_Config::singleton();
 245     $defaults = [
 246       'dataSource' => 'CRM_Import_DataSource_CSV',
 247       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 248       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 249       'fieldSeparator' => $config->fieldSeparator,
 250     ];
 251
 252     if ($loadeMapping = $this->get('loadedMapping')) {
 253       $this->assign('loadedMapping', $loadeMapping);
 254       $defaults['savedMapping'] = $loadeMapping;
 255     }
 256
 257     return $defaults;
 258   }
 259
 260   /**
 261    * @return array
 262    * @throws Exception
 263    */
 264   private function _getDataSources() {
 265     // Hmm... file-system scanners don't really belong in forms...
 266     if (isset(Civi::$statics[__CLASS__]['datasources'])) {
 267       return Civi::$statics[__CLASS__]['datasources'];
 268     }
 269
 270     // Open the data source dir and scan it for class files
 271     global $civicrm_root;
 272     $dataSourceDir = $civicrm_root . DIRECTORY_SEPARATOR . 'CRM' . DIRECTORY_SEPARATOR . 'Import' . DIRECTORY_SEPARATOR . 'DataSource' . DIRECTORY_SEPARATOR;
 273     $dataSources = [];
 274     if (!is_dir($dataSourceDir)) {
 275       $this->invalidConfig("Import DataSource directory $dataSourceDir does not exist");
 276     }
 277     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 278       $this->invalidConfig("Unable to access DataSource directory $dataSourceDir");
 279     }
 280
 281     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 282       $fileType = filetype($dataSourceDir . $dataSourceFile);
 283       $matches = [];
 284       if (($fileType == 'file' || $fileType == 'link') &&
 285         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 286       ) {
 287         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 288         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 289         $object = new $dataSourceClass();
 290         $info = $object->getInfo();
 291         if ($object->checkPermission()) {
 292           $dataSources[$dataSourceClass] = $info['title'];
 293         }
 294       }
 295     }
 296     closedir($dataSourceHandle);
 297
 298     Civi::$statics[__CLASS__]['datasources'] = $dataSources;
 299     return $dataSources;
 300   }
 301
 302   /**
 303    * Call the DataSource's postProcess method.
 304    */
 305   public function postProcess() {
 306     $this->controller->resetPage('MapField');
 307
 308     if ($this->_dataSourceIsValid) {
 309       // Setup the params array
 310       $this->_params = $this->controller->exportValues($this->_name);
 311
 312       $storeParams = [
 313         'onDuplicate' => $this->exportValue('onDuplicate'),
 314         'dedupe' => $this->exportValue('dedupe'),
 315         'contactType' => $this->exportValue('contactType'),
 316         'contactSubType' => $this->exportValue('subType'),
 317         'dateFormats' => $this->exportValue('dateFormats'),
 318         'savedMapping' => $this->exportValue('savedMapping'),
 319       ];
 320
 321       foreach ($storeParams as $storeName => $value) {
 322         $this->set($storeName, $value);
 323       }
 324       $this->set('disableUSPS', !empty($this->_params['disableUSPS']));
 325
 326       $this->set('dataSource', $this->_params['dataSource']);
 327       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 328
 329       $session = CRM_Core_Session::singleton();
 330       $session->set('dateTypes', $storeParams['dateFormats']);
 331
 332       // Get the PEAR::DB object
 333       $dao = new CRM_Core_DAO();
 334       $db = $dao->getDatabaseConnection();
 335
 336       //hack to prevent multiple tables.
 337       $this->_params['import_table_name'] = $this->get('importTableName');
 338       if (!$this->_params['import_table_name']) {
 339         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 340       }
 341
 342       $this->_dataSourceClass->postProcess($this->_params, $db, $this);
 343
 344       // We should have the data in the DB now, parse it
 345       $importTableName = $this->get('importTableName');
 346       $fieldNames = $this->_prepareImportTable($db, $importTableName);
 347       $mapper = [];
 348
 349       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 350       $parser->setMaxLinesToProcess(100);
 351       $parser->run($importTableName,
 352         $mapper,
 353         CRM_Import_Parser::MODE_MAPFIELD,
 354         $storeParams['contactType'],
 355         $fieldNames['pk'],
 356         $fieldNames['status'],
 357         CRM_Import_Parser::DUPLICATE_SKIP,
 358         NULL, NULL, FALSE,
 359         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 360         $storeParams['contactSubType'],
 361         $storeParams['dedupe']
 362       );
 363
 364       // add all the necessary variables to the form
 365       $parser->set($this);
 366     }
 367     else {
 368       $this->invalidConfig("Invalid DataSource on form post. This shouldn't happen!");
 369     }
 370   }
 371
 372   /**
 373    * Add a PK and status column to the import table so we can track our progress.
 374    * Returns the name of the primary key and status columns
 375    *
 376    * @param $db
 377    * @param string $importTableName
 378    *
 379    * @return array
 380    */
 381   private function _prepareImportTable($db, $importTableName) {
 382     /* TODO: Add a check for an existing _status field;
 383      *  if it exists, create __status instead and return that
 384      */
 385
 386     $statusFieldName = '_status';
 387     $primaryKeyName = '_id';
 388
 389     $this->set('primaryKeyName', $primaryKeyName);
 390     $this->set('statusFieldName', $statusFieldName);
 391
 392     /* Make sure the PK is always last! We rely on this later.
 393      * Should probably stop doing that at some point, but it
 394      * would require moving to associative arrays rather than
 395      * relying on numerical order of the fields. This could in
 396      * turn complicate matters for some DataSources, which
 397      * would also not be good. Decisions, decisions...
 398      */
 399
 400     $alterQuery = "ALTER TABLE $importTableName
 401                        ADD COLUMN $statusFieldName VARCHAR(32)
 402                             DEFAULT 'NEW' NOT NULL,
 403                        ADD COLUMN ${statusFieldName}Msg TEXT,
 404                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 405                                AUTO_INCREMENT";
 406     $db->query($alterQuery);
 407
 408     return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
 409   }
 410
 411   /**
 412    * General function for handling invalid configuration.
 413    *
 414    * I was going to statusBounce them all but when I tested I was 'bouncing' to weird places
 415    * whereas throwing an exception gave no behaviour change. So, I decided to centralise
 416    * and we can 'flip the switch' later.
 417    *
 418    * @param $message
 419    *
 420    * @throws \CRM_Core_Exception
 421    */
 422   protected function invalidConfig($message) {
 423     throw new CRM_Core_Exception($message);
 424   }
 425
 426   /**
 427    * Return a descriptive name for the page, used in wizard header
 428    *
 429    *
 430    * @return string
 431    */
 432   public function getTitle() {
 433     return ts('Choose Data Source');
 434   }
 435
 436 }