CRM/Custom/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
  18
  19   protected $_fileName;
  20
  21   /**
  22    * Imported file size.
  23    *
  24    * @var int
  25    */
  26   protected $_fileSize;
  27
  28   /**
  29    * Separator being used
  30    * @var string
  31    */
  32   protected $_separator;
  33
  34   /**
  35    * Total number of lines in file
  36    * @var int
  37    */
  38   protected $_lineCount;
  39
  40   /**
  41    * Whether the file has a column header or not
  42    *
  43    * @var bool
  44    */
  45   protected $_haveColumnHeader;
  46
  47   /**
  48    * @param string $fileName
  49    * @param string $separator
  50    * @param int $mapper
  51    * @param bool $skipColumnHeader
  52    * @param int|string $mode
  53    * @param int|string $contactType
  54    * @param int $onDuplicate
  55    *
  56    * @return mixed
  57    * @throws Exception
  58    */
  59   public function run(
  60     $fileName,
  61     $separator = ',',
  62     &$mapper,
  63     $skipColumnHeader = FALSE,
  64     $mode = self::MODE_PREVIEW,
  65     $contactType = self::CONTACT_INDIVIDUAL,
  66     $onDuplicate = self::DUPLICATE_SKIP
  67   ) {
  68     if (!is_array($fileName)) {
  69       throw new CRM_Core_Exception('Unable to determine import file');
  70     }
  71     $fileName = $fileName['name'];
  72
  73     switch ($contactType) {
  74       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
  75         $this->_contactType = 'Individual';
  76         break;
  77
  78       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
  79         $this->_contactType = 'Household';
  80         break;
  81
  82       case CRM_Import_Parser::CONTACT_ORGANIZATION:
  83         $this->_contactType = 'Organization';
  84     }
  85     $this->init();
  86
  87     $this->_haveColumnHeader = $skipColumnHeader;
  88
  89     $this->_separator = $separator;
  90
  91     $fd = fopen($fileName, "r");
  92     if (!$fd) {
  93       return FALSE;
  94     }
  95
  96     $this->_lineCount = $this->_warningCount = 0;
  97     $this->_invalidRowCount = $this->_validCount = 0;
  98     $this->_totalCount = $this->_conflictCount = 0;
  99
 100     $this->_errors = [];
 101     $this->_warnings = [];
 102     $this->_conflicts = [];
 103
 104     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 105
 106     if ($mode == self::MODE_MAPFIELD) {
 107       $this->_rows = [];
 108     }
 109     else {
 110       $this->_activeFieldCount = count($this->_activeFields);
 111     }
 112
 113     while (!feof($fd)) {
 114       $this->_lineCount++;
 115
 116       $values = fgetcsv($fd, 8192, $separator);
 117       if (!$values) {
 118         continue;
 119       }
 120
 121       self::encloseScrub($values);
 122
 123       // skip column header if we're not in mapfield mode
 124       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 125         $skipColumnHeader = FALSE;
 126         continue;
 127       }
 128
 129       /* trim whitespace around the values */
 130
 131       $empty = TRUE;
 132       foreach ($values as $k => $v) {
 133         $values[$k] = trim($v, " \t\r\n");
 134       }
 135
 136       if (CRM_Utils_System::isNull($values)) {
 137         continue;
 138       }
 139
 140       $this->_totalCount++;
 141
 142       if ($mode == self::MODE_MAPFIELD) {
 143         $returnCode = $this->mapField($values);
 144       }
 145       elseif ($mode == self::MODE_PREVIEW) {
 146         $returnCode = $this->preview($values);
 147       }
 148       elseif ($mode == self::MODE_SUMMARY) {
 149         $returnCode = $this->summary($values);
 150       }
 151       elseif ($mode == self::MODE_IMPORT) {
 152         $returnCode = $this->import($onDuplicate, $values);
 153       }
 154       else {
 155         $returnCode = self::ERROR;
 156       }
 157
 158       // note that a line could be valid but still produce a warning
 159       if ($returnCode & self::VALID) {
 160         $this->_validCount++;
 161         if ($mode == self::MODE_MAPFIELD) {
 162           $this->_rows[] = $values;
 163           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 164         }
 165       }
 166
 167       if ($returnCode & self::WARNING) {
 168         $this->_warningCount++;
 169         if ($this->_warningCount < $this->_maxWarningCount) {
 170           $this->_warningCount[] = $line;
 171         }
 172       }
 173
 174       if ($returnCode & self::ERROR) {
 175         $this->_invalidRowCount++;
 176         $recordNumber = $this->_lineCount;
 177         if ($this->_haveColumnHeader) {
 178           $recordNumber--;
 179         }
 180         array_unshift($values, $recordNumber);
 181         $this->_errors[] = $values;
 182       }
 183
 184       if ($returnCode & self::CONFLICT) {
 185         $this->_conflictCount++;
 186         $recordNumber = $this->_lineCount;
 187         if ($this->_haveColumnHeader) {
 188           $recordNumber--;
 189         }
 190         array_unshift($values, $recordNumber);
 191         $this->_conflicts[] = $values;
 192       }
 193
 194       if ($returnCode & self::DUPLICATE) {
 195         if ($returnCode & self::MULTIPLE_DUPE) {
 196           /* TODO: multi-dupes should be counted apart from singles
 197            * on non-skip action */
 198         }
 199         $this->_duplicateCount++;
 200         $recordNumber = $this->_lineCount;
 201         if ($this->_haveColumnHeader) {
 202           $recordNumber--;
 203         }
 204         array_unshift($values, $recordNumber);
 205         $this->_duplicates[] = $values;
 206         if ($onDuplicate != self::DUPLICATE_SKIP) {
 207           $this->_validCount++;
 208         }
 209       }
 210
 211       // we give the derived class a way of aborting the process
 212       // note that the return code could be multiple code or'ed together
 213       if ($returnCode & self::STOP) {
 214         break;
 215       }
 216
 217       // if we are done processing the maxNumber of lines, break
 218       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 219         break;
 220       }
 221     }
 222
 223     fclose($fd);
 224
 225     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 226       $customHeaders = $mapper;
 227
 228       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 229       foreach ($customHeaders as $key => $value) {
 230         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 231           $customHeaders[$key] = $customfields[$id][0];
 232         }
 233       }
 234       if ($this->_invalidRowCount) {
 235         // removed view url for invlaid contacts
 236         $headers = array_merge([
 237           ts('Line Number'),
 238           ts('Reason'),
 239         ], $customHeaders);
 240         $this->_errorFileName = self::errorFileName(self::ERROR);
 241         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 242       }
 243       if ($this->_conflictCount) {
 244         $headers = array_merge([
 245           ts('Line Number'),
 246           ts('Reason'),
 247         ], $customHeaders);
 248         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 249         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 250       }
 251       if ($this->_duplicateCount) {
 252         $headers = array_merge([
 253           ts('Line Number'),
 254           ts('View Activity History URL'),
 255         ], $customHeaders);
 256
 257         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 258         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 259       }
 260     }
 261     return $this->fini();
 262   }
 263
 264   /**
 265    * Given a list of the importable field keys that the user has selected
 266    * set the active fields array to this list
 267    *
 268    * @param array $fieldKeys mapped array of values
 269    *
 270    * @return void
 271    */
 272   public function setActiveFields($fieldKeys) {
 273     $this->_activeFieldCount = count($fieldKeys);
 274     foreach ($fieldKeys as $key) {
 275       if (empty($this->_fields[$key])) {
 276         $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
 277       }
 278       else {
 279         $this->_activeFields[] = clone($this->_fields[$key]);
 280       }
 281     }
 282   }
 283
 284   /**
 285    * Format the field values for input to the api.
 286    *
 287    * @return array
 288    *   (reference ) associative array of name/value pairs
 289    */
 290   public function &getActiveFieldParams() {
 291     $params = [];
 292     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 293       if (isset($this->_activeFields[$i]->_value)
 294         && !isset($params[$this->_activeFields[$i]->_name])
 295         && !isset($this->_activeFields[$i]->_related)
 296       ) {
 297         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 298       }
 299     }
 300     return $params;
 301   }
 302
 303   /**
 304    * Store parser values.
 305    *
 306    * @param CRM_Core_Session $store
 307    *
 308    * @param int $mode
 309    *
 310    * @return void
 311    */
 312   public function set($store, $mode = self::MODE_SUMMARY) {
 313     $store->set('fileSize', $this->_fileSize);
 314     $store->set('lineCount', $this->_lineCount);
 315     $store->set('separator', $this->_separator);
 316     $store->set('fields', $this->getSelectValues());
 317     $store->set('fieldTypes', $this->getSelectTypes());
 318
 319     $store->set('headerPatterns', $this->getHeaderPatterns());
 320     $store->set('dataPatterns', $this->getDataPatterns());
 321     $store->set('columnCount', $this->_activeFieldCount);
 322     $store->set('_entity', $this->_entity);
 323     $store->set('totalRowCount', $this->_totalCount);
 324     $store->set('validRowCount', $this->_validCount);
 325     $store->set('invalidRowCount', $this->_invalidRowCount);
 326     $store->set('conflictRowCount', $this->_conflictCount);
 327
 328     switch ($this->_contactType) {
 329       case 'Individual':
 330         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 331         break;
 332
 333       case 'Household':
 334         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 335         break;
 336
 337       case 'Organization':
 338         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 339     }
 340
 341     if ($this->_invalidRowCount) {
 342       $store->set('errorsFileName', $this->_errorFileName);
 343     }
 344     if ($this->_conflictCount) {
 345       $store->set('conflictsFileName', $this->_conflictFileName);
 346     }
 347     if (isset($this->_rows) && !empty($this->_rows)) {
 348       $store->set('dataValues', $this->_rows);
 349     }
 350
 351     if ($mode == self::MODE_IMPORT) {
 352       $store->set('duplicateRowCount', $this->_duplicateCount);
 353       if ($this->_duplicateCount) {
 354         $store->set('duplicatesFileName', $this->_duplicateFileName);
 355       }
 356     }
 357   }
 358
 359 }