CRM/Contact/Import/Form/DataSource.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.5                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2009.                                       |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36 /**
  37  * This class delegates to the chosen DataSource to grab the data to be
  38  *  imported.
  39  */
  40 class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form {
  41
  42   private $_dataSource;
  43
  44   private $_dataSourceIsValid = FALSE;
  45
  46   private $_dataSourceClassFile;
  47
  48   private $_dataSourceClass;
  49
  50   /**
  51    * Function to set variables up before form is built
  52    *
  53    * @return void
  54    * @access public
  55    */
  56   public function preProcess() {
  57
  58     //Test database user privilege to create table(Temporary) CRM-4725
  59     $errorScope = CRM_Core_TemporaryErrorScope::ignoreException();
  60     $daoTestPrivilege = new CRM_Core_DAO;
  61     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_one(test int) ENGINE=InnoDB");
  62     $daoTestPrivilege->query("CREATE TEMPORARY TABLE import_job_permission_two(test int) ENGINE=InnoDB");
  63     $daoTestPrivilege->query("DROP TABLE IF EXISTS import_job_permission_one, import_job_permission_two");
  64     unset($errorScope);
  65
  66     if ($daoTestPrivilege->_lastError) {
  67       CRM_Core_Error::fatal(ts('Database Configuration Error: Insufficient permissions. Import requires that the CiviCRM database user has permission to create temporary tables. Contact your site administrator for assistance.'));
  68     }
  69
  70     $results    = array();
  71     $config     = CRM_Core_Config::singleton();
  72     $handler    = opendir($config->uploadDir);
  73     $errorFiles = array('sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch');
  74
  75     while ($file = readdir($handler)) {
  76       if ($file != '.' && $file != '..' &&
  77         in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file)
  78       ) {
  79         $results[] = $file;
  80       }
  81     }
  82     closedir($handler);
  83     if (!empty($results)) {
  84       CRM_Core_Error::fatal(ts('<b>%1</b> file(s) in %2 directory are not writable. Listed file(s) might be used during the import to log the errors occurred during Import process. Contact your site administrator for assistance.', array(1 => implode(', ', $results), 2 => $config->uploadDir)));
  85     }
  86
  87     $this->_dataSourceIsValid = FALSE;
  88     $this->_dataSource = CRM_Utils_Request::retrieve(
  89       'dataSource',
  90       'String',
  91       CRM_Core_DAO::$_nullObject,
  92       FALSE,
  93       NULL,
  94       'GET'
  95     );
  96
  97     $this->_params = $this->controller->exportValues($this->_name);
  98     if (!$this->_dataSource) {
  99       //considering dataSource as base criteria instead of hidden_dataSource.
 100       $this->_dataSource = CRM_Utils_Array::value('dataSource',
 101         $_POST,
 102         CRM_Utils_Array::value('dataSource',
 103           $this->_params
 104         )
 105       );
 106       $this->assign('showOnlyDataSourceFormPane', FALSE);
 107     }
 108     else {
 109       $this->assign('showOnlyDataSourceFormPane', TRUE);
 110     }
 111
 112     if (strpos($this->_dataSource, 'CRM_Import_DataSource_') === 0) {
 113       $this->_dataSourceIsValid = TRUE;
 114       $this->assign('showDataSourceFormPane', TRUE);
 115       $dataSourcePath = explode('_', $this->_dataSource);
 116       $templateFile = "CRM/Contact/Import/Form/" . $dataSourcePath[3] . ".tpl";
 117       $this->assign('dataSourceFormTemplateFile', $templateFile);
 118     }
 119   }
 120
 121   /**
 122    * Function to actually build the form
 123    *
 124    * @return void
 125    * @access public
 126    */
 127
 128   public function buildQuickForm() {
 129
 130     // If there's a dataSource in the query string, we need to load
 131     // the form from the chosen DataSource class
 132     if ($this->_dataSourceIsValid) {
 133       $this->_dataSourceClassFile = str_replace('_', '/', $this->_dataSource) . ".php";
 134       require_once $this->_dataSourceClassFile;
 135       $this->_dataSourceClass = new $this->_dataSource;
 136       $this->_dataSourceClass->buildQuickForm( $this );
 137     }
 138
 139     // Get list of data sources and display them as options
 140     $dataSources = $this->_getDataSources();
 141
 142     $this->assign('urlPath', "civicrm/import");
 143     $this->assign('urlPathVar', 'snippet=4');
 144
 145     $this->add('select', 'dataSource', ts('Data Source'), $dataSources, TRUE,
 146       array('onchange' => 'buildDataSourceFormBlock(this.value);')
 147     );
 148
 149     // duplicate handling options
 150     $duplicateOptions = array();
 151     $duplicateOptions[] = $this->createElement('radio',
 152       NULL, NULL, ts('Skip'), CRM_Import_Parser::DUPLICATE_SKIP
 153     );
 154     $duplicateOptions[] = $this->createElement('radio',
 155       NULL, NULL, ts('Update'), CRM_Import_Parser::DUPLICATE_UPDATE
 156     );
 157     $duplicateOptions[] = $this->createElement('radio',
 158       NULL, NULL, ts('Fill'), CRM_Import_Parser::DUPLICATE_FILL
 159     );
 160     $duplicateOptions[] = $this->createElement('radio',
 161       NULL, NULL, ts('No Duplicate Checking'), CRM_Import_Parser::DUPLICATE_NOCHECK
 162     );
 163
 164     $this->addGroup($duplicateOptions, 'onDuplicate',
 165       ts('For Duplicate Contacts')
 166     );
 167
 168     $mappingArray = CRM_Core_BAO_Mapping::getMappings(CRM_Core_OptionGroup::getValue('mapping_type',
 169         'Import Contact',
 170         'name'
 171       ));
 172
 173     $this->assign('savedMapping', $mappingArray);
 174     $this->addElement('select', 'savedMapping', ts('Mapping Option'), array('' => ts('- select -')) + $mappingArray);
 175
 176
 177     $js = array('onClick' => "buildSubTypes();buildDedupeRules();");
 178     // contact types option
 179     $contactOptions = array();
 180     if (CRM_Contact_BAO_ContactType::isActive('Individual')) {
 181       $contactOptions[] = $this->createElement('radio',
 182         NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL, $js
 183       );
 184     }
 185     if (CRM_Contact_BAO_ContactType::isActive('Household')) {
 186       $contactOptions[] = $this->createElement('radio',
 187         NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD, $js
 188       );
 189     }
 190     if (CRM_Contact_BAO_ContactType::isActive('Organization')) {
 191       $contactOptions[] = $this->createElement('radio',
 192         NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION, $js
 193       );
 194     }
 195
 196     $this->addGroup($contactOptions, 'contactType',
 197       ts('Contact Type')
 198     );
 199
 200     $this->addElement('select', 'subType', ts('Subtype'));
 201     $this->addElement('select', 'dedupe', ts('Dedupe Rule'));
 202
 203     CRM_Core_Form_Date::buildAllowedDateFormats($this);
 204
 205     $config = CRM_Core_Config::singleton();
 206     $geoCode = FALSE;
 207     if (!empty($config->geocodeMethod)) {
 208       $geoCode = TRUE;
 209       $this->addElement('checkbox', 'doGeocodeAddress', ts('Lookup mapping info during import?'));
 210     }
 211     $this->assign('geoCode', $geoCode);
 212
 213     $this->addElement('text', 'fieldSeparator', ts('Import Field Separator'), array('size' => 2));
 214
 215     $this->addButtons(array(
 216         array(
 217           'type' => 'upload',
 218           'name' => ts('Continue >>'),
 219           'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
 220           'isDefault' => TRUE,
 221         ),
 222         array(
 223           'type' => 'cancel',
 224           'name' => ts('Cancel'),
 225         ),
 226       )
 227     );
 228   }
 229
 230   /**
 231    * This virtual function is used to set the default values of
 232    * various form elements
 233    *
 234    * access        public
 235    *
 236    * @return array reference to the array of default values
 237    *
 238    */
 239   /**
 240    * @return array
 241    */
 242   function setDefaultValues() {
 243     $config = CRM_Core_Config::singleton();
 244     $defaults = array(
 245       'dataSource' => 'CRM_Import_DataSource_CSV',
 246       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
 247       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
 248       'fieldSeparator' => $config->fieldSeparator,
 249     );
 250
 251     if ($loadeMapping = $this->get('loadedMapping')) {
 252       $this->assign('loadedMapping', $loadeMapping);
 253       $defaults['savedMapping'] = $loadeMapping;
 254     }
 255
 256     return $defaults;
 257   }
 258
 259   /**
 260    * @return array
 261    * @throws Exception
 262    */
 263   private function _getDataSources() {
 264     // Open the data source dir and scan it for class files
 265     $config        = CRM_Core_Config::singleton();
 266     $dataSourceDir = $config->importDataSourceDir;
 267     $dataSources   = array();
 268     if (!is_dir($dataSourceDir)) {
 269       CRM_Core_Error::fatal("Import DataSource directory $dataSourceDir does not exist");
 270     }
 271     if (!$dataSourceHandle = opendir($dataSourceDir)) {
 272       CRM_Core_Error::fatal("Unable to access DataSource directory $dataSourceDir");
 273     }
 274
 275     while (($dataSourceFile = readdir($dataSourceHandle)) !== FALSE) {
 276       $fileType = filetype($dataSourceDir . $dataSourceFile);
 277       $matches = array();
 278       if (($fileType == 'file' || $fileType == 'link') &&
 279         preg_match('/^(.+)\.php$/', $dataSourceFile, $matches)
 280       ) {
 281         $dataSourceClass = "CRM_Import_DataSource_" . $matches[1];
 282         require_once $dataSourceDir . DIRECTORY_SEPARATOR . $dataSourceFile;
 283         $object = new $dataSourceClass;
 284         $info   = $object->getInfo();
 285         $dataSources[$dataSourceClass] = $info['title'];
 286       }
 287     }
 288     closedir($dataSourceHandle);
 289     return $dataSources;
 290   }
 291
 292   /**
 293    * Call the DataSource's postProcess method to take over
 294    * and then setup some common data structures for the next step
 295    *
 296    * @return void
 297    * @access public
 298    */
 299   public function postProcess() {
 300     $this->controller->resetPage('MapField');
 301
 302     if ($this->_dataSourceIsValid) {
 303       // Setup the params array
 304       $this->_params = $this->controller->exportValues($this->_name);
 305
 306       $storeParams = array(
 307         'onDuplicate' => 'onDuplicate',
 308         'dedupe' => 'dedupe',
 309         'contactType' => 'contactType',
 310         'contactSubType' => 'subType',
 311         'dateFormats' => 'dateFormats',
 312         'savedMapping' => 'savedMapping',
 313       );
 314
 315       foreach ($storeParams as $storeName => $storeValueName) {
 316         $$storeName = $this->exportValue($storeValueName);
 317         $this->set($storeName, $$storeName);
 318       }
 319
 320       $this->set('dataSource', $this->_params['dataSource']);
 321       $this->set('skipColumnHeader', CRM_Utils_Array::value('skipColumnHeader', $this->_params));
 322
 323       $session = CRM_Core_Session::singleton();
 324       $session->set('dateTypes', $dateFormats);
 325
 326       // Get the PEAR::DB object
 327       $dao = new CRM_Core_DAO();
 328       $db = $dao->getDatabaseConnection();
 329
 330       //hack to prevent multiple tables.
 331       $this->_params['import_table_name'] = $this->get('importTableName');
 332       if (!$this->_params['import_table_name']) {
 333         $this->_params['import_table_name'] = 'civicrm_import_job_' . md5(uniqid(rand(), TRUE));
 334       }
 335
 336       $this->_dataSourceClass->postProcess( $this->_params, $db, $this );
 337
 338       // We should have the data in the DB now, parse it
 339       $importTableName = $this->get('importTableName');
 340       $fieldNames      = $this->_prepareImportTable($db, $importTableName);
 341       $mapper          = array();
 342
 343       $parser = new CRM_Contact_Import_Parser_Contact($mapper);
 344       $parser->setMaxLinesToProcess(100);
 345       $parser->run($importTableName,
 346         $mapper,
 347         CRM_Import_Parser::MODE_MAPFIELD,
 348         $contactType,
 349         $fieldNames['pk'],
 350         $fieldNames['status'],
 351         CRM_Import_Parser::DUPLICATE_SKIP,
 352         NULL, NULL, FALSE,
 353         CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
 354         $contactSubType,
 355         $dedupe
 356       );
 357
 358       // add all the necessary variables to the form
 359       $parser->set($this);
 360     }
 361     else {
 362       CRM_Core_Error::fatal("Invalid DataSource on form post. This shouldn't happen!");
 363     }
 364   }
 365
 366   /**
 367    * Add a PK and status column to the import table so we can track our progress
 368    * Returns the name of the primary key and status columns
 369    *
 370    * @param $db
 371    * @param $importTableName
 372    *
 373    * @return array
 374    * @access private
 375    */
 376   private function _prepareImportTable($db, $importTableName) {
 377     /* TODO: Add a check for an existing _status field;
 378          *  if it exists, create __status instead and return that
 379          */
 380
 381     $statusFieldName = '_status';
 382     $primaryKeyName = '_id';
 383
 384     $this->set('primaryKeyName', $primaryKeyName);
 385     $this->set('statusFieldName', $statusFieldName);
 386
 387     /* Make sure the PK is always last! We rely on this later.
 388          * Should probably stop doing that at some point, but it
 389          * would require moving to associative arrays rather than
 390          * relying on numerical order of the fields. This could in
 391          * turn complicate matters for some DataSources, which
 392          * would also not be good. Decisions, decisions...
 393          */
 394
 395     $alterQuery = "ALTER TABLE $importTableName
 396                        ADD COLUMN $statusFieldName VARCHAR(32)
 397                             DEFAULT 'NEW' NOT NULL,
 398                        ADD COLUMN ${statusFieldName}Msg TEXT,
 399                        ADD COLUMN $primaryKeyName INT PRIMARY KEY NOT NULL
 400                                AUTO_INCREMENT";
 401     $db->query($alterQuery);
 402
 403     return array('status' => $statusFieldName, 'pk' => $primaryKeyName);
 404   }
 405
 406   /**
 407    * Return a descriptive name for the page, used in wizard header
 408    *
 409    *
 410    * @return string
 411    * @access public
 412    */
 413   public function getTitle() {
 414     return ts('Choose Data Source');
 415   }
 416 }
 417