CRM/Custom/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**
  40    * #@+
  41    * @var integer
  42    */
  43
  44   /**
  45    * Imported file size
  46    * @var int
  47    */
  48   protected $_fileSize;
  49
  50   /**
  51    * Separator being used
  52    * @var string
  53    */
  54   protected $_separator;
  55
  56   /**
  57    * Total number of lines in file
  58    * @var int
  59    */
  60   protected $_lineCount;
  61
  62   /**
  63    * Whether the file has a column header or not
  64    *
  65    * @var bool
  66    */
  67   protected $_haveColumnHeader;
  68
  69   /**
  70    * @param string $fileName
  71    * @param string $separator
  72    * @param int $mapper
  73    * @param bool $skipColumnHeader
  74    * @param int|string $mode
  75    * @param int|string $contactType
  76    * @param int $onDuplicate
  77    *
  78    * @return mixed
  79    * @throws Exception
  80    */
  81   public function run(
  82     $fileName,
  83     $separator = ',',
  84     &$mapper,
  85     $skipColumnHeader = FALSE,
  86     $mode = self::MODE_PREVIEW,
  87     $contactType = self::CONTACT_INDIVIDUAL,
  88     $onDuplicate = self::DUPLICATE_SKIP
  89   ) {
  90     if (!is_array($fileName)) {
  91       CRM_Core_Error::fatal();
  92     }
  93     $fileName = $fileName['name'];
  94
  95     switch ($contactType) {
  96       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
  97         $this->_contactType = 'Individual';
  98         break;
  99
 100       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
 101         $this->_contactType = 'Household';
 102         break;
 103
 104       case CRM_Import_Parser::CONTACT_ORGANIZATION:
 105         $this->_contactType = 'Organization';
 106     }
 107     $this->init();
 108
 109     $this->_haveColumnHeader = $skipColumnHeader;
 110
 111     $this->_separator = $separator;
 112
 113     $fd = fopen($fileName, "r");
 114     if (!$fd) {
 115       return FALSE;
 116     }
 117
 118     $this->_lineCount = $this->_warningCount = 0;
 119     $this->_invalidRowCount = $this->_validCount = 0;
 120     $this->_totalCount = $this->_conflictCount = 0;
 121
 122     $this->_errors = [];
 123     $this->_warnings = [];
 124     $this->_conflicts = [];
 125
 126     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 127
 128     if ($mode == self::MODE_MAPFIELD) {
 129       $this->_rows = [];
 130     }
 131     else {
 132       $this->_activeFieldCount = count($this->_activeFields);
 133     }
 134
 135     while (!feof($fd)) {
 136       $this->_lineCount++;
 137
 138       $values = fgetcsv($fd, 8192, $separator);
 139       if (!$values) {
 140         continue;
 141       }
 142
 143       self::encloseScrub($values);
 144
 145       // skip column header if we're not in mapfield mode
 146       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 147         $skipColumnHeader = FALSE;
 148         continue;
 149       }
 150
 151       /* trim whitespace around the values */
 152
 153       $empty = TRUE;
 154       foreach ($values as $k => $v) {
 155         $values[$k] = trim($v, " \t\r\n");
 156       }
 157
 158       if (CRM_Utils_System::isNull($values)) {
 159         continue;
 160       }
 161
 162       $this->_totalCount++;
 163
 164       if ($mode == self::MODE_MAPFIELD) {
 165         $returnCode = $this->mapField($values);
 166       }
 167       elseif ($mode == self::MODE_PREVIEW) {
 168         $returnCode = $this->preview($values);
 169       }
 170       elseif ($mode == self::MODE_SUMMARY) {
 171         $returnCode = $this->summary($values);
 172       }
 173       elseif ($mode == self::MODE_IMPORT) {
 174         $returnCode = $this->import($onDuplicate, $values);
 175       }
 176       else {
 177         $returnCode = self::ERROR;
 178       }
 179
 180       // note that a line could be valid but still produce a warning
 181       if ($returnCode & self::VALID) {
 182         $this->_validCount++;
 183         if ($mode == self::MODE_MAPFIELD) {
 184           $this->_rows[] = $values;
 185           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 186         }
 187       }
 188
 189       if ($returnCode & self::WARNING) {
 190         $this->_warningCount++;
 191         if ($this->_warningCount < $this->_maxWarningCount) {
 192           $this->_warningCount[] = $line;
 193         }
 194       }
 195
 196       if ($returnCode & self::ERROR) {
 197         $this->_invalidRowCount++;
 198         $recordNumber = $this->_lineCount;
 199         if ($this->_haveColumnHeader) {
 200           $recordNumber--;
 201         }
 202         array_unshift($values, $recordNumber);
 203         $this->_errors[] = $values;
 204       }
 205
 206       if ($returnCode & self::CONFLICT) {
 207         $this->_conflictCount++;
 208         $recordNumber = $this->_lineCount;
 209         if ($this->_haveColumnHeader) {
 210           $recordNumber--;
 211         }
 212         array_unshift($values, $recordNumber);
 213         $this->_conflicts[] = $values;
 214       }
 215
 216       if ($returnCode & self::DUPLICATE) {
 217         if ($returnCode & self::MULTIPLE_DUPE) {
 218           /* TODO: multi-dupes should be counted apart from singles
 219            * on non-skip action */
 220         }
 221         $this->_duplicateCount++;
 222         $recordNumber = $this->_lineCount;
 223         if ($this->_haveColumnHeader) {
 224           $recordNumber--;
 225         }
 226         array_unshift($values, $recordNumber);
 227         $this->_duplicates[] = $values;
 228         if ($onDuplicate != self::DUPLICATE_SKIP) {
 229           $this->_validCount++;
 230         }
 231       }
 232
 233       // we give the derived class a way of aborting the process
 234       // note that the return code could be multiple code or'ed together
 235       if ($returnCode & self::STOP) {
 236         break;
 237       }
 238
 239       // if we are done processing the maxNumber of lines, break
 240       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 241         break;
 242       }
 243     }
 244
 245     fclose($fd);
 246
 247     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 248       $customHeaders = $mapper;
 249
 250       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 251       foreach ($customHeaders as $key => $value) {
 252         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 253           $customHeaders[$key] = $customfields[$id][0];
 254         }
 255       }
 256       if ($this->_invalidRowCount) {
 257         // removed view url for invlaid contacts
 258         $headers = array_merge([
 259           ts('Line Number'),
 260           ts('Reason'),
 261         ], $customHeaders);
 262         $this->_errorFileName = self::errorFileName(self::ERROR);
 263         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 264       }
 265       if ($this->_conflictCount) {
 266         $headers = array_merge([
 267           ts('Line Number'),
 268           ts('Reason'),
 269         ], $customHeaders);
 270         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 271         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 272       }
 273       if ($this->_duplicateCount) {
 274         $headers = array_merge([
 275           ts('Line Number'),
 276           ts('View Activity History URL'),
 277         ], $customHeaders);
 278
 279         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 280         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 281       }
 282     }
 283     return $this->fini();
 284   }
 285
 286   /**
 287    * Given a list of the importable field keys that the user has selected
 288    * set the active fields array to this list
 289    *
 290    * @param array $fieldKeys mapped array of values
 291    *
 292    * @return void
 293    */
 294   public function setActiveFields($fieldKeys) {
 295     $this->_activeFieldCount = count($fieldKeys);
 296     foreach ($fieldKeys as $key) {
 297       if (empty($this->_fields[$key])) {
 298         $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
 299       }
 300       else {
 301         $this->_activeFields[] = clone($this->_fields[$key]);
 302       }
 303     }
 304   }
 305
 306   /**
 307    * Format the field values for input to the api.
 308    *
 309    * @return array
 310    *   (reference ) associative array of name/value pairs
 311    */
 312   public function &getActiveFieldParams() {
 313     $params = [];
 314     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 315       if (isset($this->_activeFields[$i]->_value)
 316         && !isset($params[$this->_activeFields[$i]->_name])
 317         && !isset($this->_activeFields[$i]->_related)
 318       ) {
 319         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 320       }
 321     }
 322     return $params;
 323   }
 324
 325   /**
 326    * Store parser values.
 327    *
 328    * @param CRM_Core_Session $store
 329    *
 330    * @param int $mode
 331    *
 332    * @return void
 333    */
 334   public function set($store, $mode = self::MODE_SUMMARY) {
 335     $store->set('fileSize', $this->_fileSize);
 336     $store->set('lineCount', $this->_lineCount);
 337     $store->set('seperator', $this->_separator);
 338     $store->set('fields', $this->getSelectValues());
 339     $store->set('fieldTypes', $this->getSelectTypes());
 340
 341     $store->set('headerPatterns', $this->getHeaderPatterns());
 342     $store->set('dataPatterns', $this->getDataPatterns());
 343     $store->set('columnCount', $this->_activeFieldCount);
 344     $store->set('_entity', $this->_entity);
 345     $store->set('totalRowCount', $this->_totalCount);
 346     $store->set('validRowCount', $this->_validCount);
 347     $store->set('invalidRowCount', $this->_invalidRowCount);
 348     $store->set('conflictRowCount', $this->_conflictCount);
 349
 350     switch ($this->_contactType) {
 351       case 'Individual':
 352         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 353         break;
 354
 355       case 'Household':
 356         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 357         break;
 358
 359       case 'Organization':
 360         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 361     }
 362
 363     if ($this->_invalidRowCount) {
 364       $store->set('errorsFileName', $this->_errorFileName);
 365     }
 366     if ($this->_conflictCount) {
 367       $store->set('conflictsFileName', $this->_conflictFileName);
 368     }
 369     if (isset($this->_rows) && !empty($this->_rows)) {
 370       $store->set('dataValues', $this->_rows);
 371     }
 372
 373     if ($mode == self::MODE_IMPORT) {
 374       $store->set('duplicateRowCount', $this->_duplicateCount);
 375       if ($this->_duplicateCount) {
 376         $store->set('duplicatesFileName', $this->_duplicateFileName);
 377       }
 378     }
 379   }
 380
 381 }