CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2009 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33
  34 /**
  35  * This class delegates to the chosen DataSource to grab the data to be imported.
  36  */
  37 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  38
  39   private $_dataSource;
  40
  41   private $_dataSourceIsValid = FALSE;
  42
  43   private $_dataSourceClassFile;
  44
  45   private $_dataSourceClass;
  46
  47   /**
  48    * Set variables up before form is built.
  49    */
  50   public function preProcess() {
  51
  52     //Test database user privilege to create table(Temporary) CRM-4725
  53     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  54     $daoTestPrivilege = new CRM_Core_DAO();
  55     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_one(test int) ENGINE=InnoDB");
  56     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_two(test int) ENGINE=InnoDB");
  57     $daoTestPrivilege->query("DROP TEMPORARY TABLE IF EXISTS import_job_permission_one, import_job_permission_two");
  58     unset($errorScope);
  59
  60     if ($daoTestPrivilege->_lastError) {
  61       CRM_Core_Error::fatal(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  62     }
  63
  64     $results = [];
  65     $config = CRM_Core_Config::singleton();
  66     $handler = opendir($config->uploadDir);
  67     $errorFiles = ['sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch'];
  68
  69     // check for post max size avoid when called twice
  70     $snippet = CRM_Utils_Array::value('snippet', $_GET, 0);
  71     if (empty($snippet)) {
  72       CRM_Utils_Number::formatUnitSize(ini_get('post_max_size'), TRUE);
  73     }
  74
  75     while ($file = readdir($handler)) {
  76       if ($file != '.' && $file != '..' &&
  77         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  78       ) {
  79         $results[] = $file;
  80       }
  81     }
  82     closedir($handler);
  83     if (!empty($results)) {
  84       CRM_Core_Error::fatal(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', [
  85         1 => implode(', ', $results),
  86         2 => $config->uploadDir,
  87       ]));
  88     }
  89
  90     $this->_dataSourceIsValid = FALSE;
  91     $this->_dataSource = CRM_Utils_Request::retrieve(
  92       'dataSource',
  93       'String',
  94       CRM_Core_DAO::$_nullObject,
  95       FALSE,
  96       NULL,
  97       'GET'
  98     );
  99
 100     $this->_params = $this->controller->exportValues($this->_name);
 101     if (!$this->_dataSource) {
 102       //considering dataSource as base criteria instead of hidden_dataSource.
 103       $this->_dataSource = CRM_Utils_Array::value('dataSource',
 104         $_POST,
 105         CRM_Utils_Array::value('dataSource',
 106           $this->_params
 107         )
 108       );
 109       $this->assign('showOnlyDataSourceFormPane', FALSE);
 110     }
 111     else {
 112       $this->assign('showOnlyDataSourceFormPane', TRUE);
 113     }
 114
 115     $dataSources = $this->_getDataSources();
 116     if ($this->_dataSource && isset($dataSources[$this->_dataSource])) {
 117       $this->_dataSourceIsValid = TRUE;
 118       $this->assign('showDataSourceFormPane', TRUE);
 119       $dataSourcePath = explode('_', $this->_dataSource);
 120       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 121       $this->assign('dataSourceFormTemplateFile', $templateFile);
 122     }
 123     elseif ($this->_dataSource) {
 124       throw new \CRM_Core_Exception("Invalid data source");
 125     }
 126   }
 127
 128   /**
 129    * Build the form object.
 130    */
 131   public function buildQuickForm() {
 132
 133     // If there's a dataSource in the query string, we need to load
 134     // the form from the chosen DataSource class
 135     if ($this->_dataSourceIsValid) {
 136       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 137       require_once $this->_dataSourceClassFile;
 138       $this->_dataSourceClass = new $this->_dataSource();
 139       $this->_dataSourceClass->buildQuickForm($this);
 140     }
 141
 142     // Get list of data sources and display them as options
 143     $dataSources = $this->_getDataSources();
 144
 145     $this->assign('urlPath', "civicrm/import");
 146     $this->assign('urlPathVar', 'snippet=4');
 147
 148     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 149       ['onchange' => 'buildDataSourceFormBlock(this.value);']
 150     );
 151
 152     // duplicate handling options
 153     $duplicateOptions = [];
 154     $duplicateOptions[] = $this->createElement('radio',
 155       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 156     );
 157     $duplicateOptions[] = $this->createElement('radio',
 158       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 159     );
 160     $duplicateOptions[] = $this->createElement('radio',
 161       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 162     );
 163     $duplicateOptions[] = $this->createElement('radio',
 164       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 165     );
 166
 167     $this->addGroup($duplicateOptions, 'onDuplicate',
 168       ts('For Duplicate Contacts')
 169     );
 170
 171     $mappingArray = CRM_Core_BAO_Mapping::getMappings('Import Contact');
 172
 173     $this->assign('savedMapping', $mappingArray);
 174     $this->addElement('select', 'savedMapping', ts('Mapping Option'), ['' => ts('- select -')] + $mappingArray);
 175
 176     $js = ['onClick' => "buildSubTypes();buildDedupeRules();"];
 177     // contact types option
 178     $contactOptions = [];
 179     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 180       $contactOptions[] = $this->createElement('radio',
 181         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 182       );
 183     }
 184     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 185       $contactOptions[] = $this->createElement('radio',
 186         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 187       );
 188     }
 189     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 190       $contactOptions[] = $this->createElement('radio',
 191         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 192       );
 193     }
 194
 195     $this->addGroup($contactOptions, 'contactType',
 196       ts('Contact Type')
 197     );
 198
 199     $this->addElement('select', 'subType', ts('Subtype'));
 200     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 201
 202     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 203
 204     $config = CRM_Core_Config::singleton();
 205     $geoCode = FALSE;
 206     if (CRM_Utils_GeocodeProvider::getUsableClassName()) {
 207       $geoCode = TRUE;
 208       $this->addElement('checkbox', 'doGeocodeAddress', ts('Geocode addresses during import?'));
 209     }
 210     $this->assign('geoCode', $geoCode);
 211
 212     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), ['size' => 2]);
 213
 214     if (Civi::settings()->get('address_standardization_provider') == 'USPS') {
 215       $this->addElement('checkbox', 'disableUSPS', ts('Disable USPS address validation during import?'));
 216     }
 217
 218     $this->addButtons([
 219       [
 220         'type' => 'upload',
 221         'name' => ts('Continue'),
 222         'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 223         'isDefault' => TRUE,
 224       ],
 225       [
 226         'type' => 'cancel',
 227         'name' => ts('Cancel'),
 228       ],
 229     ]);
 230   }
 231
 232   /**
 233    * Set the default values of various form elements.
 234    *
 235    * access        public
 236    *
 237    * @return array
 238    *   reference to the array of default values
 239    */
 240   public function setDefaultValues() {
 241     $config = CRM_Core_Config::singleton();
 242     $defaults = [
 243       'dataSource' => 'CRM_Import_DataSource_CSV',
 244       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 245       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 246       'fieldSeparator' => $config->fieldSeparator,
 247     ];
 248
 249     if ($loadeMapping = $this->get('loadedMapping')) {
 250       $this->assign('loadedMapping', $loadeMapping);
 251       $defaults['savedMapping'] = $loadeMapping;
 252     }
 253
 254     return $defaults;
 255   }
 256
 257   /**
 258    * @return array
 259    * @throws Exception
 260    */
 261   private function _getDataSources() {
 262     // Hmm... file-system scanners don't really belong in forms...
 263     if (isset(Civi::$statics[__CLASS__]['datasources'])) {
 264       return Civi::$statics[__CLASS__]['datasources'];
 265     }
 266
 267     // Open the data source dir and scan it for class files
 268     global $civicrm_root;
 269     $dataSourceDir = $civicrm_root . DIRECTORY_SEPARATOR . 'CRM' . DIRECTORY_SEPARATOR . 'Import' . DIRECTORY_SEPARATOR . 'DataSource' . DIRECTORY_SEPARATOR;
 270     $dataSources = [];
 271     if (!is_dir($dataSourceDir)) {
 272       CRM_Core_Error::fatal("Import DataSource directory $dataSourceDir does not exist");
 273     }
 274     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 275       CRM_Core_Error::fatal("Unable to access DataSource directory $dataSourceDir");
 276     }
 277
 278     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 279       $fileType = filetype($dataSourceDir . $dataSourceFile);
 280       $matches = [];
 281       if (($fileType == 'file' || $fileType == 'link') &&
 282         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 283       ) {
 284         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 285         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 286         $object = new $dataSourceClass();
 287         $info = $object->getInfo();
 288         if ($object->checkPermission()) {
 289           $dataSources[$dataSourceClass] = $info['title'];
 290         }
 291       }
 292     }
 293     closedir($dataSourceHandle);
 294
 295     Civi::$statics[__CLASS__]['datasources'] = $dataSources;
 296     return $dataSources;
 297   }
 298
 299   /**
 300    * Call the DataSource's postProcess method.
 301    */
 302   public function postProcess() {
 303     $this->controller->resetPage('MapField');
 304
 305     if ($this->_dataSourceIsValid) {
 306       // Setup the params array
 307       $this->_params = $this->controller->exportValues($this->_name);
 308
 309       $storeParams = [
 310         'onDuplicate' => 'onDuplicate',
 311         'dedupe' => 'dedupe',
 312         'contactType' => 'contactType',
 313         'contactSubType' => 'subType',
 314         'dateFormats' => 'dateFormats',
 315         'savedMapping' => 'savedMapping',
 316       ];
 317
 318       foreach ($storeParams as $storeName => $storeValueName) {
 319         $$storeName = $this->exportValue($storeValueName);
 320         $this->set($storeName, $$storeName);
 321       }
 322       $this->set('disableUSPS', !empty($this->_params['disableUSPS']));
 323
 324       $this->set('dataSource', $this->_params['dataSource']);
 325       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 326
 327       $session = CRM_Core_Session::singleton();
 328       $session->set('dateTypes', $dateFormats);
 329
 330       // Get the PEAR::DB object
 331       $dao = new CRM_Core_DAO();
 332       $db = $dao->getDatabaseConnection();
 333
 334       //hack to prevent multiple tables.
 335       $this->_params['import_table_name'] = $this->get('importTableName');
 336       if (!$this->_params['import_table_name']) {
 337         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 338       }
 339
 340       $this->_dataSourceClass->postProcess($this->_params, $db, $this);
 341
 342       // We should have the data in the DB now, parse it
 343       $importTableName = $this->get('importTableName');
 344       $fieldNames = $this->_prepareImportTable($db, $importTableName);
 345       $mapper = [];
 346
 347       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 348       $parser->setMaxLinesToProcess(100);
 349       $parser->run($importTableName,
 350         $mapper,
 351         CRM_Import_Parser::MODE_MAPFIELD,
 352         $contactType,
 353         $fieldNames['pk'],
 354         $fieldNames['status'],
 355         CRM_Import_Parser::DUPLICATE_SKIP,
 356         NULL, NULL, FALSE,
 357         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 358         $contactSubType,
 359         $dedupe
 360       );
 361
 362       // add all the necessary variables to the form
 363       $parser->set($this);
 364     }
 365     else {
 366       CRM_Core_Error::fatal("Invalid DataSource on form post. This shouldn't happen!");
 367     }
 368   }
 369
 370   /**
 371    * Add a PK and status column to the import table so we can track our progress.
 372    * Returns the name of the primary key and status columns
 373    *
 374    * @param $db
 375    * @param string $importTableName
 376    *
 377    * @return array
 378    */
 379   private function _prepareImportTable($db, $importTableName) {
 380     /* TODO: Add a check for an existing _status field;
 381      *  if it exists, create __status instead and return that
 382      */
 383
 384     $statusFieldName = '_status';
 385     $primaryKeyName = '_id';
 386
 387     $this->set('primaryKeyName', $primaryKeyName);
 388     $this->set('statusFieldName', $statusFieldName);
 389
 390     /* Make sure the PK is always last! We rely on this later.
 391      * Should probably stop doing that at some point, but it
 392      * would require moving to associative arrays rather than
 393      * relying on numerical order of the fields. This could in
 394      * turn complicate matters for some DataSources, which
 395      * would also not be good. Decisions, decisions...
 396      */
 397
 398     $alterQuery = "ALTER TABLE $importTableName
 399                        ADD COLUMN $statusFieldName VARCHAR(32)
 400                             DEFAULT 'NEW' NOT NULL,
 401                        ADD COLUMN ${statusFieldName}Msg TEXT,
 402                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 403                                AUTO_INCREMENT";
 404     $db->query($alterQuery);
 405
 406     return ['status' => $statusFieldName, 'pk' => $primaryKeyName];
 407   }
 408
 409   /**
 410    * Return a descriptive name for the page, used in wizard header
 411    *
 412    *
 413    * @return string
 414    */
 415   public function getTitle() {
 416     return ts('Choose Data Source');
 417   }
 418
 419 }