CRM/Event/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Event_Import_Parser extends CRM_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**
  40    * Imported file size.
  41    *
  42    * @var int
  43    */
  44   protected $_fileSize;
  45
  46   /**
  47    * Separator being used.
  48    *
  49    * @var string
  50    */
  51   protected $_seperator;
  52
  53   /**
  54    * Total number of lines in file.
  55    *
  56    * @var int
  57    */
  58   protected $_lineCount;
  59
  60   /**
  61    * Whether the file has a column header or not
  62    *
  63    * @var bool
  64    */
  65   protected $_haveColumnHeader;
  66
  67   /**
  68    * @param string $fileName
  69    * @param string $seperator
  70    * @param $mapper
  71    * @param bool $skipColumnHeader
  72    * @param int $mode
  73    * @param int $contactType
  74    * @param int $onDuplicate
  75    *
  76    * @return mixed
  77    * @throws Exception
  78    */
  79   public function run(
  80     $fileName,
  81     $seperator = ',',
  82     &$mapper,
  83     $skipColumnHeader = FALSE,
  84     $mode = self::MODE_PREVIEW,
  85     $contactType = self::CONTACT_INDIVIDUAL,
  86     $onDuplicate = self::DUPLICATE_SKIP
  87   ) {
  88     if (!is_array($fileName)) {
  89       CRM_Core_Error::fatal();
  90     }
  91     $fileName = $fileName['name'];
  92
  93     switch ($contactType) {
  94       case self::CONTACT_INDIVIDUAL:
  95         $this->_contactType = 'Individual';
  96         break;
  97
  98       case self::CONTACT_HOUSEHOLD:
  99         $this->_contactType = 'Household';
 100         break;
 101
 102       case self::CONTACT_ORGANIZATION:
 103         $this->_contactType = 'Organization';
 104     }
 105
 106     $this->init();
 107
 108     $this->_haveColumnHeader = $skipColumnHeader;
 109
 110     $this->_seperator = $seperator;
 111
 112     $fd = fopen($fileName, "r");
 113     if (!$fd) {
 114       return FALSE;
 115     }
 116
 117     $this->_lineCount = $this->_warningCount = 0;
 118     $this->_invalidRowCount = $this->_validCount = 0;
 119     $this->_totalCount = $this->_conflictCount = 0;
 120
 121     $this->_errors = [];
 122     $this->_warnings = [];
 123     $this->_conflicts = [];
 124
 125     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 126
 127     if ($mode == self::MODE_MAPFIELD) {
 128       $this->_rows = [];
 129     }
 130     else {
 131       $this->_activeFieldCount = count($this->_activeFields);
 132     }
 133
 134     while (!feof($fd)) {
 135       $this->_lineCount++;
 136
 137       $values = fgetcsv($fd, 8192, $seperator);
 138       if (!$values) {
 139         continue;
 140       }
 141
 142       self::encloseScrub($values);
 143
 144       // skip column header if we're not in mapfield mode
 145       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 146         $skipColumnHeader = FALSE;
 147         continue;
 148       }
 149
 150       /* trim whitespace around the values */
 151
 152       $empty = TRUE;
 153       foreach ($values as $k => $v) {
 154         $values[$k] = trim($v, " \t\r\n");
 155       }
 156
 157       if (CRM_Utils_System::isNull($values)) {
 158         continue;
 159       }
 160
 161       $this->_totalCount++;
 162
 163       if ($mode == self::MODE_MAPFIELD) {
 164         $returnCode = $this->mapField($values);
 165       }
 166       elseif ($mode == self::MODE_PREVIEW) {
 167         $returnCode = $this->preview($values);
 168       }
 169       elseif ($mode == self::MODE_SUMMARY) {
 170         $returnCode = $this->summary($values);
 171       }
 172       elseif ($mode == self::MODE_IMPORT) {
 173         $returnCode = $this->import($onDuplicate, $values);
 174       }
 175       else {
 176         $returnCode = self::ERROR;
 177       }
 178
 179       // note that a line could be valid but still produce a warning
 180       if ($returnCode & self::VALID) {
 181         $this->_validCount++;
 182         if ($mode == self::MODE_MAPFIELD) {
 183           $this->_rows[] = $values;
 184           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 185         }
 186       }
 187
 188       if ($returnCode & self::WARNING) {
 189         $this->_warningCount++;
 190         if ($this->_warningCount < $this->_maxWarningCount) {
 191           $this->_warningCount[] = $line;
 192         }
 193       }
 194
 195       if ($returnCode & self::ERROR) {
 196         $this->_invalidRowCount++;
 197         $recordNumber = $this->_lineCount;
 198         if ($this->_haveColumnHeader) {
 199           $recordNumber--;
 200         }
 201         array_unshift($values, $recordNumber);
 202         $this->_errors[] = $values;
 203       }
 204
 205       if ($returnCode & self::CONFLICT) {
 206         $this->_conflictCount++;
 207         $recordNumber = $this->_lineCount;
 208         if ($this->_haveColumnHeader) {
 209           $recordNumber--;
 210         }
 211         array_unshift($values, $recordNumber);
 212         $this->_conflicts[] = $values;
 213       }
 214
 215       if ($returnCode & self::DUPLICATE) {
 216         if ($returnCode & self::MULTIPLE_DUPE) {
 217           /* TODO: multi-dupes should be counted apart from singles
 218            * on non-skip action */
 219         }
 220         $this->_duplicateCount++;
 221         $recordNumber = $this->_lineCount;
 222         if ($this->_haveColumnHeader) {
 223           $recordNumber--;
 224         }
 225         array_unshift($values, $recordNumber);
 226         $this->_duplicates[] = $values;
 227         if ($onDuplicate != self::DUPLICATE_SKIP) {
 228           $this->_validCount++;
 229         }
 230       }
 231
 232       // we give the derived class a way of aborting the process
 233       // note that the return code could be multiple code or'ed together
 234       if ($returnCode & self::STOP) {
 235         break;
 236       }
 237
 238       // if we are done processing the maxNumber of lines, break
 239       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 240         break;
 241       }
 242     }
 243
 244     fclose($fd);
 245
 246     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 247       $customHeaders = $mapper;
 248
 249       $customfields = CRM_Core_BAO_CustomField::getFields('Participant');
 250       foreach ($customHeaders as $key => $value) {
 251         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 252           $customHeaders[$key] = $customfields[$id][0];
 253         }
 254       }
 255
 256       if ($this->_invalidRowCount) {
 257         // removed view url for invlaid contacts
 258         $headers = array_merge([
 259           ts('Line Number'),
 260           ts('Reason'),
 261         ], $customHeaders);
 262         $this->_errorFileName = self::errorFileName(self::ERROR);
 263         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 264       }
 265       if ($this->_conflictCount) {
 266         $headers = array_merge([
 267           ts('Line Number'),
 268           ts('Reason'),
 269         ], $customHeaders);
 270         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 271         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 272       }
 273       if ($this->_duplicateCount) {
 274         $headers = array_merge([
 275           ts('Line Number'),
 276           ts('View Participant URL'),
 277         ], $customHeaders);
 278
 279         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 280         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 281       }
 282     }
 283     return $this->fini();
 284   }
 285
 286   /**
 287    * Given a list of the importable field keys that the user has selected
 288    * set the active fields array to this list
 289    *
 290    * @param $fieldKeys array mapped array of values
 291    *
 292    * @return void
 293    */
 294   public function setActiveFields($fieldKeys) {
 295     $this->_activeFieldCount = count($fieldKeys);
 296     foreach ($fieldKeys as $key) {
 297       if (empty($this->_fields[$key])) {
 298         $this->_activeFields[] = new CRM_Event_Import_Field('', ts('- do not import -'));
 299       }
 300       else {
 301         $this->_activeFields[] = clone($this->_fields[$key]);
 302       }
 303     }
 304   }
 305
 306   /**
 307    * Format the field values for input to the api.
 308    *
 309    * @return array
 310    *   (reference ) associative array of name/value pairs
 311    */
 312   public function &getActiveFieldParams() {
 313     $params = [];
 314     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 315       if (isset($this->_activeFields[$i]->_value)
 316         && !isset($params[$this->_activeFields[$i]->_name])
 317         && !isset($this->_activeFields[$i]->_related)
 318       ) {
 319
 320         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 321       }
 322     }
 323     return $params;
 324   }
 325
 326   /**
 327    * @param string $name
 328    * @param $title
 329    * @param int $type
 330    * @param string $headerPattern
 331    * @param string $dataPattern
 332    */
 333   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 334     if (empty($name)) {
 335       $this->_fields['doNotImport'] = new CRM_Event_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 336     }
 337     else {
 338
 339       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 340       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 341       if (!array_key_exists($name, $tempField)) {
 342         $this->_fields[$name] = new CRM_Event_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 343       }
 344       else {
 345         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 346           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 347         );
 348       }
 349     }
 350   }
 351
 352   /**
 353    * Store parser values.
 354    *
 355    * @param CRM_Core_Session $store
 356    *
 357    * @param int $mode
 358    *
 359    * @return void
 360    */
 361   public function set($store, $mode = self::MODE_SUMMARY) {
 362     $store->set('fileSize', $this->_fileSize);
 363     $store->set('lineCount', $this->_lineCount);
 364     $store->set('seperator', $this->_seperator);
 365     $store->set('fields', $this->getSelectValues());
 366     $store->set('fieldTypes', $this->getSelectTypes());
 367
 368     $store->set('headerPatterns', $this->getHeaderPatterns());
 369     $store->set('dataPatterns', $this->getDataPatterns());
 370     $store->set('columnCount', $this->_activeFieldCount);
 371
 372     $store->set('totalRowCount', $this->_totalCount);
 373     $store->set('validRowCount', $this->_validCount);
 374     $store->set('invalidRowCount', $this->_invalidRowCount);
 375     $store->set('conflictRowCount', $this->_conflictCount);
 376
 377     switch ($this->_contactType) {
 378       case 'Individual':
 379         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 380         break;
 381
 382       case 'Household':
 383         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 384         break;
 385
 386       case 'Organization':
 387         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 388     }
 389
 390     if ($this->_invalidRowCount) {
 391       $store->set('errorsFileName', $this->_errorFileName);
 392     }
 393     if ($this->_conflictCount) {
 394       $store->set('conflictsFileName', $this->_conflictFileName);
 395     }
 396     if (isset($this->_rows) && !empty($this->_rows)) {
 397       $store->set('dataValues', $this->_rows);
 398     }
 399
 400     if ($mode == self::MODE_IMPORT) {
 401       $store->set('duplicateRowCount', $this->_duplicateCount);
 402       if ($this->_duplicateCount) {
 403         $store->set('duplicatesFileName', $this->_duplicateFileName);
 404       }
 405     }
 406   }
 407
 408   /**
 409    * Export data to a CSV file.
 410    *
 411    * @param string $fileName
 412    * @param array $header
 413    * @param array $data
 414    *
 415    * @return void
 416    */
 417   public static function exportCSV($fileName, $header, $data) {
 418     $output = [];
 419     $fd = fopen($fileName, 'w');
 420
 421     foreach ($header as $key => $value) {
 422       $header[$key] = "\"$value\"";
 423     }
 424     $config = CRM_Core_Config::singleton();
 425     $output[] = implode($config->fieldSeparator, $header);
 426
 427     foreach ($data as $datum) {
 428       foreach ($datum as $key => $value) {
 429         if (is_array($value)) {
 430           foreach ($value[0] as $k1 => $v1) {
 431             if ($k1 == 'location_type_id') {
 432               continue;
 433             }
 434             $datum[$k1] = $v1;
 435           }
 436         }
 437         else {
 438           $datum[$key] = "\"$value\"";
 439         }
 440       }
 441       $output[] = implode($config->fieldSeparator, $datum);
 442     }
 443     fwrite($fd, implode("\n", $output));
 444     fclose($fd);
 445   }
 446
 447 }