CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.6                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2009.                                       |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36 /**
  37  * This class delegates to the chosen DataSource to grab the data to be
  38  *  imported.
  39  */
  40 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  41
  42   private $_dataSource;
  43
  44   private $_dataSourceIsValid = FALSE;
  45
  46   private $_dataSourceClassFile;
  47
  48   private $_dataSourceClass;
  49
  50   /**
  51    * Set variables up before form is built
  52    *
  53    * @return void
  54    */
  55   public function preProcess() {
  56
  57     //Test database user privilege to create table(Temporary) CRM-4725
  58     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  59     $daoTestPrivilege = new CRM_Core_DAO;
  60     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_one(test int) ENGINE=InnoDB");
  61     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_two(test int) ENGINE=InnoDB");
  62     $daoTestPrivilege->query("DROP TABLE IF EXISTS import_job_permission_one, import_job_permission_two");
  63     unset($errorScope);
  64
  65     if ($daoTestPrivilege->_lastError) {
  66       CRM_Core_Error::fatal(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  67     }
  68
  69     $results = array();
  70     $config = CRM_Core_Config::singleton();
  71     $handler = opendir($config->uploadDir);
  72     $errorFiles = array('sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch');
  73
  74     // check for post max size avoid when called twice
  75     $snippet = CRM_Utils_Array::value('snippet', $_GET, 0);
  76     if (empty($snippet)) {
  77       CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE);
  78     }
  79
  80     while ($file = readdir($handler)) {
  81       if ($file != '.' && $file != '..' &&
  82         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  83       ) {
  84         $results[] = $file;
  85       }
  86     }
  87     closedir($handler);
  88     if (!empty($results)) {
  89       CRM_Core_Error::fatal(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', array(
  90             1 => implode(', ', $results),
  91             2 => $config->uploadDir
  92           )));
  93     }
  94
  95     $this->_dataSourceIsValid = FALSE;
  96     $this->_dataSource = CRM_Utils_Request::retrieve(
  97       'dataSource',
  98       'String',
  99       CRM_Core_DAO::$_nullObject,
 100       FALSE,
 101       NULL,
 102       'GET'
 103     );
 104
 105     $this->_params = $this->controller->exportValues($this->_name);
 106     if (!$this->_dataSource) {
 107       //considering dataSource as base criteria instead of hidden_dataSource.
 108       $this->_dataSource = CRM_Utils_Array::value('dataSource',
 109         $_POST,
 110         CRM_Utils_Array::value('dataSource',
 111           $this->_params
 112         )
 113       );
 114       $this->assign('showOnlyDataSourceFormPane', FALSE);
 115     }
 116     else {
 117       $this->assign('showOnlyDataSourceFormPane', TRUE);
 118     }
 119
 120     if (strpos($this->_dataSource, 'CRM_Import_DataSource_') === 0) {
 121       $this->_dataSourceIsValid = TRUE;
 122       $this->assign('showDataSourceFormPane', TRUE);
 123       $dataSourcePath = explode('_', $this->_dataSource);
 124       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 125       $this->assign('dataSourceFormTemplateFile', $templateFile);
 126     }
 127   }
 128
 129   /**
 130    * Build the form object
 131    *
 132    * @return void
 133    */
 134
 135   public function buildQuickForm() {
 136
 137     // If there's a dataSource in the query string, we need to load
 138     // the form from the chosen DataSource class
 139     if ($this->_dataSourceIsValid) {
 140       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 141       require_once $this->_dataSourceClassFile;
 142       $this->_dataSourceClass = new $this->_dataSource;
 143       $this->_dataSourceClass->buildQuickForm($this);
 144     }
 145
 146     // Get list of data sources and display them as options
 147     $dataSources = $this->_getDataSources();
 148
 149     $this->assign('urlPath', "civicrm/import");
 150     $this->assign('urlPathVar', 'snippet=4');
 151
 152     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 153       array('onchange' => 'buildDataSourceFormBlock(this.value);')
 154     );
 155
 156     // duplicate handling options
 157     $duplicateOptions = array();
 158     $duplicateOptions[] = $this->createElement('radio',
 159       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 160     );
 161     $duplicateOptions[] = $this->createElement('radio',
 162       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 163     );
 164     $duplicateOptions[] = $this->createElement('radio',
 165       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 166     );
 167     $duplicateOptions[] = $this->createElement('radio',
 168       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 169     );
 170
 171     $this->addGroup($duplicateOptions, 'onDuplicate',
 172       ts('For Duplicate Contacts')
 173     );
 174
 175     $mappingArray = CRM_Core_BAO_Mapping::getMappings(CRM_Core_OptionGroup::getValue('mapping_type',
 176       'Import Contact',
 177       'name'
 178     ));
 179
 180     $this->assign('savedMapping', $mappingArray);
 181     $this->addElement('select', 'savedMapping', ts('Mapping Option'), array('' => ts('- select -')) + $mappingArray);
 182
 183
 184     $js = array('onClick' => "buildSubTypes();buildDedupeRules();");
 185     // contact types option
 186     $contactOptions = array();
 187     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 188       $contactOptions[] = $this->createElement('radio',
 189         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 190       );
 191     }
 192     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 193       $contactOptions[] = $this->createElement('radio',
 194         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 195       );
 196     }
 197     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 198       $contactOptions[] = $this->createElement('radio',
 199         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 200       );
 201     }
 202
 203     $this->addGroup($contactOptions, 'contactType',
 204       ts('Contact Type')
 205     );
 206
 207     $this->addElement('select', 'subType', ts('Subtype'));
 208     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 209
 210     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 211
 212     $config = CRM_Core_Config::singleton();
 213     $geoCode = FALSE;
 214     if (!empty($config->geocodeMethod)) {
 215       $geoCode = TRUE;
 216       $this->addElement('checkbox', 'doGeocodeAddress', ts('Lookup mapping info during import?'));
 217     }
 218     $this->assign('geoCode', $geoCode);
 219
 220     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), array('size' => 2));
 221
 222     $this->addButtons(array(
 223         array(
 224           'type' => 'upload',
 225           'name' => ts('Continue'),
 226           'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 227           'isDefault' => TRUE,
 228         ),
 229         array(
 230           'type' => 'cancel',
 231           'name' => ts('Cancel'),
 232         ),
 233       )
 234     );
 235   }
 236
 237   /**
 238    * This virtual function is used to set the default values of
 239    * various form elements
 240    *
 241    * access        public
 242    *
 243    * @return array
 244    *   reference to the array of default values
 245    */
 246   /**
 247    * @return array
 248    */
 249   public function setDefaultValues() {
 250     $config = CRM_Core_Config::singleton();
 251     $defaults = array(
 252       'dataSource' => 'CRM_Import_DataSource_CSV',
 253       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 254       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 255       'fieldSeparator' => $config->fieldSeparator,
 256     );
 257
 258     if ($loadeMapping = $this->get('loadedMapping')) {
 259       $this->assign('loadedMapping', $loadeMapping);
 260       $defaults['savedMapping'] = $loadeMapping;
 261     }
 262
 263     return $defaults;
 264   }
 265
 266   /**
 267    * @return array
 268    * @throws Exception
 269    */
 270   private function _getDataSources() {
 271     // Open the data source dir and scan it for class files
 272     $config = CRM_Core_Config::singleton();
 273     $dataSourceDir = $config->importDataSourceDir;
 274     $dataSources = array();
 275     if (!is_dir($dataSourceDir)) {
 276       CRM_Core_Error::fatal("Import DataSource directory $dataSourceDir does not exist");
 277     }
 278     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 279       CRM_Core_Error::fatal("Unable to access DataSource directory $dataSourceDir");
 280     }
 281
 282     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 283       $fileType = filetype($dataSourceDir . $dataSourceFile);
 284       $matches = array();
 285       if (($fileType == 'file' || $fileType == 'link') &&
 286         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 287       ) {
 288         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 289         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 290         $object = new $dataSourceClass;
 291         $info = $object->getInfo();
 292         $dataSources[$dataSourceClass] = $info['title'];
 293       }
 294     }
 295     closedir($dataSourceHandle);
 296     return $dataSources;
 297   }
 298
 299   /**
 300    * Call the DataSource's postProcess method to take over
 301    * and then setup some common data structures for the next step
 302    *
 303    * @return void
 304    */
 305   public function postProcess() {
 306     $this->controller->resetPage('MapField');
 307
 308     if ($this->_dataSourceIsValid) {
 309       // Setup the params array
 310       $this->_params = $this->controller->exportValues($this->_name);
 311
 312       $storeParams = array(
 313         'onDuplicate' => 'onDuplicate',
 314         'dedupe' => 'dedupe',
 315         'contactType' => 'contactType',
 316         'contactSubType' => 'subType',
 317         'dateFormats' => 'dateFormats',
 318         'savedMapping' => 'savedMapping',
 319       );
 320
 321       foreach ($storeParams as $storeName => $storeValueName) {
 322         $$storeName = $this->exportValue($storeValueName);
 323         $this->set($storeName, $$storeName);
 324       }
 325
 326       $this->set('dataSource', $this->_params['dataSource']);
 327       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 328
 329       $session = CRM_Core_Session::singleton();
 330       $session->set('dateTypes', $dateFormats);
 331
 332       // Get the PEAR::DB object
 333       $dao = new CRM_Core_DAO();
 334       $db = $dao->getDatabaseConnection();
 335
 336       //hack to prevent multiple tables.
 337       $this->_params['import_table_name'] = $this->get('importTableName');
 338       if (!$this->_params['import_table_name']) {
 339         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 340       }
 341
 342       $this->_dataSourceClass->postProcess($this->_params, $db, $this);
 343
 344       // We should have the data in the DB now, parse it
 345       $importTableName = $this->get('importTableName');
 346       $fieldNames = $this->_prepareImportTable($db, $importTableName);
 347       $mapper = array();
 348
 349       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 350       $parser->setMaxLinesToProcess(100);
 351       $parser->run($importTableName,
 352         $mapper,
 353         CRM_Import_Parser::MODE_MAPFIELD,
 354         $contactType,
 355         $fieldNames['pk'],
 356         $fieldNames['status'],
 357         CRM_Import_Parser::DUPLICATE_SKIP,
 358         NULL, NULL, FALSE,
 359         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 360         $contactSubType,
 361         $dedupe
 362       );
 363
 364       // add all the necessary variables to the form
 365       $parser->set($this);
 366     }
 367     else {
 368       CRM_Core_Error::fatal("Invalid DataSource on form post. This shouldn't happen!");
 369     }
 370   }
 371
 372   /**
 373    * Add a PK and status column to the import table so we can track our progress
 374    * Returns the name of the primary key and status columns
 375    *
 376    * @param $db
 377    * @param string $importTableName
 378    *
 379    * @return array
 380    */
 381   private function _prepareImportTable($db, $importTableName) {
 382     /* TODO: Add a check for an existing _status field;
 383          *  if it exists, create __status instead and return that
 384          */
 385
 386     $statusFieldName = '_status';
 387     $primaryKeyName = '_id';
 388
 389     $this->set('primaryKeyName', $primaryKeyName);
 390     $this->set('statusFieldName', $statusFieldName);
 391
 392     /* Make sure the PK is always last! We rely on this later.
 393          * Should probably stop doing that at some point, but it
 394          * would require moving to associative arrays rather than
 395          * relying on numerical order of the fields. This could in
 396          * turn complicate matters for some DataSources, which
 397          * would also not be good. Decisions, decisions...
 398          */
 399
 400     $alterQuery = "ALTER TABLE $importTableName
 401                        ADD COLUMN $statusFieldName VARCHAR(32)
 402                             DEFAULT 'NEW' NOT NULL,
 403                        ADD COLUMN ${statusFieldName}Msg TEXT,
 404                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 405                                AUTO_INCREMENT";
 406     $db->query($alterQuery);
 407
 408     return array('status' => $statusFieldName, 'pk' => $primaryKeyName);
 409   }
 410
 411   /**
 412    * Return a descriptive name for the page, used in wizard header
 413    *
 414    *
 415    * @return string
 416    */
 417   public function getTitle() {
 418     return ts('Choose Data Source');
 419   }
 420 }