CRM/Custom/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  * $Id$
  17  *
  18  */
  19 abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
  20
  21   protected $_fileName;
  22
  23   /**
  24    * Imported file size.
  25    *
  26    * @var int
  27    */
  28   protected $_fileSize;
  29
  30   /**
  31    * Separator being used
  32    * @var string
  33    */
  34   protected $_separator;
  35
  36   /**
  37    * Total number of lines in file
  38    * @var int
  39    */
  40   protected $_lineCount;
  41
  42   /**
  43    * Whether the file has a column header or not
  44    *
  45    * @var bool
  46    */
  47   protected $_haveColumnHeader;
  48
  49   /**
  50    * @param string $fileName
  51    * @param string $separator
  52    * @param int $mapper
  53    * @param bool $skipColumnHeader
  54    * @param int|string $mode
  55    * @param int|string $contactType
  56    * @param int $onDuplicate
  57    *
  58    * @return mixed
  59    * @throws Exception
  60    */
  61   public function run(
  62     $fileName,
  63     $separator = ',',
  64     &$mapper,
  65     $skipColumnHeader = FALSE,
  66     $mode = self::MODE_PREVIEW,
  67     $contactType = self::CONTACT_INDIVIDUAL,
  68     $onDuplicate = self::DUPLICATE_SKIP
  69   ) {
  70     if (!is_array($fileName)) {
  71       CRM_Core_Error::fatal();
  72     }
  73     $fileName = $fileName['name'];
  74
  75     switch ($contactType) {
  76       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
  77         $this->_contactType = 'Individual';
  78         break;
  79
  80       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
  81         $this->_contactType = 'Household';
  82         break;
  83
  84       case CRM_Import_Parser::CONTACT_ORGANIZATION:
  85         $this->_contactType = 'Organization';
  86     }
  87     $this->init();
  88
  89     $this->_haveColumnHeader = $skipColumnHeader;
  90
  91     $this->_separator = $separator;
  92
  93     $fd = fopen($fileName, "r");
  94     if (!$fd) {
  95       return FALSE;
  96     }
  97
  98     $this->_lineCount = $this->_warningCount = 0;
  99     $this->_invalidRowCount = $this->_validCount = 0;
 100     $this->_totalCount = $this->_conflictCount = 0;
 101
 102     $this->_errors = [];
 103     $this->_warnings = [];
 104     $this->_conflicts = [];
 105
 106     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 107
 108     if ($mode == self::MODE_MAPFIELD) {
 109       $this->_rows = [];
 110     }
 111     else {
 112       $this->_activeFieldCount = count($this->_activeFields);
 113     }
 114
 115     while (!feof($fd)) {
 116       $this->_lineCount++;
 117
 118       $values = fgetcsv($fd, 8192, $separator);
 119       if (!$values) {
 120         continue;
 121       }
 122
 123       self::encloseScrub($values);
 124
 125       // skip column header if we're not in mapfield mode
 126       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 127         $skipColumnHeader = FALSE;
 128         continue;
 129       }
 130
 131       /* trim whitespace around the values */
 132
 133       $empty = TRUE;
 134       foreach ($values as $k => $v) {
 135         $values[$k] = trim($v, " \t\r\n");
 136       }
 137
 138       if (CRM_Utils_System::isNull($values)) {
 139         continue;
 140       }
 141
 142       $this->_totalCount++;
 143
 144       if ($mode == self::MODE_MAPFIELD) {
 145         $returnCode = $this->mapField($values);
 146       }
 147       elseif ($mode == self::MODE_PREVIEW) {
 148         $returnCode = $this->preview($values);
 149       }
 150       elseif ($mode == self::MODE_SUMMARY) {
 151         $returnCode = $this->summary($values);
 152       }
 153       elseif ($mode == self::MODE_IMPORT) {
 154         $returnCode = $this->import($onDuplicate, $values);
 155       }
 156       else {
 157         $returnCode = self::ERROR;
 158       }
 159
 160       // note that a line could be valid but still produce a warning
 161       if ($returnCode & self::VALID) {
 162         $this->_validCount++;
 163         if ($mode == self::MODE_MAPFIELD) {
 164           $this->_rows[] = $values;
 165           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 166         }
 167       }
 168
 169       if ($returnCode & self::WARNING) {
 170         $this->_warningCount++;
 171         if ($this->_warningCount < $this->_maxWarningCount) {
 172           $this->_warningCount[] = $line;
 173         }
 174       }
 175
 176       if ($returnCode & self::ERROR) {
 177         $this->_invalidRowCount++;
 178         $recordNumber = $this->_lineCount;
 179         if ($this->_haveColumnHeader) {
 180           $recordNumber--;
 181         }
 182         array_unshift($values, $recordNumber);
 183         $this->_errors[] = $values;
 184       }
 185
 186       if ($returnCode & self::CONFLICT) {
 187         $this->_conflictCount++;
 188         $recordNumber = $this->_lineCount;
 189         if ($this->_haveColumnHeader) {
 190           $recordNumber--;
 191         }
 192         array_unshift($values, $recordNumber);
 193         $this->_conflicts[] = $values;
 194       }
 195
 196       if ($returnCode & self::DUPLICATE) {
 197         if ($returnCode & self::MULTIPLE_DUPE) {
 198           /* TODO: multi-dupes should be counted apart from singles
 199            * on non-skip action */
 200         }
 201         $this->_duplicateCount++;
 202         $recordNumber = $this->_lineCount;
 203         if ($this->_haveColumnHeader) {
 204           $recordNumber--;
 205         }
 206         array_unshift($values, $recordNumber);
 207         $this->_duplicates[] = $values;
 208         if ($onDuplicate != self::DUPLICATE_SKIP) {
 209           $this->_validCount++;
 210         }
 211       }
 212
 213       // we give the derived class a way of aborting the process
 214       // note that the return code could be multiple code or'ed together
 215       if ($returnCode & self::STOP) {
 216         break;
 217       }
 218
 219       // if we are done processing the maxNumber of lines, break
 220       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 221         break;
 222       }
 223     }
 224
 225     fclose($fd);
 226
 227     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 228       $customHeaders = $mapper;
 229
 230       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 231       foreach ($customHeaders as $key => $value) {
 232         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 233           $customHeaders[$key] = $customfields[$id][0];
 234         }
 235       }
 236       if ($this->_invalidRowCount) {
 237         // removed view url for invlaid contacts
 238         $headers = array_merge([
 239           ts('Line Number'),
 240           ts('Reason'),
 241         ], $customHeaders);
 242         $this->_errorFileName = self::errorFileName(self::ERROR);
 243         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 244       }
 245       if ($this->_conflictCount) {
 246         $headers = array_merge([
 247           ts('Line Number'),
 248           ts('Reason'),
 249         ], $customHeaders);
 250         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 251         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 252       }
 253       if ($this->_duplicateCount) {
 254         $headers = array_merge([
 255           ts('Line Number'),
 256           ts('View Activity History URL'),
 257         ], $customHeaders);
 258
 259         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 260         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 261       }
 262     }
 263     return $this->fini();
 264   }
 265
 266   /**
 267    * Given a list of the importable field keys that the user has selected
 268    * set the active fields array to this list
 269    *
 270    * @param array $fieldKeys mapped array of values
 271    *
 272    * @return void
 273    */
 274   public function setActiveFields($fieldKeys) {
 275     $this->_activeFieldCount = count($fieldKeys);
 276     foreach ($fieldKeys as $key) {
 277       if (empty($this->_fields[$key])) {
 278         $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
 279       }
 280       else {
 281         $this->_activeFields[] = clone($this->_fields[$key]);
 282       }
 283     }
 284   }
 285
 286   /**
 287    * Format the field values for input to the api.
 288    *
 289    * @return array
 290    *   (reference ) associative array of name/value pairs
 291    */
 292   public function &getActiveFieldParams() {
 293     $params = [];
 294     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 295       if (isset($this->_activeFields[$i]->_value)
 296         && !isset($params[$this->_activeFields[$i]->_name])
 297         && !isset($this->_activeFields[$i]->_related)
 298       ) {
 299         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 300       }
 301     }
 302     return $params;
 303   }
 304
 305   /**
 306    * Store parser values.
 307    *
 308    * @param CRM_Core_Session $store
 309    *
 310    * @param int $mode
 311    *
 312    * @return void
 313    */
 314   public function set($store, $mode = self::MODE_SUMMARY) {
 315     $store->set('fileSize', $this->_fileSize);
 316     $store->set('lineCount', $this->_lineCount);
 317     $store->set('seperator', $this->_separator);
 318     $store->set('fields', $this->getSelectValues());
 319     $store->set('fieldTypes', $this->getSelectTypes());
 320
 321     $store->set('headerPatterns', $this->getHeaderPatterns());
 322     $store->set('dataPatterns', $this->getDataPatterns());
 323     $store->set('columnCount', $this->_activeFieldCount);
 324     $store->set('_entity', $this->_entity);
 325     $store->set('totalRowCount', $this->_totalCount);
 326     $store->set('validRowCount', $this->_validCount);
 327     $store->set('invalidRowCount', $this->_invalidRowCount);
 328     $store->set('conflictRowCount', $this->_conflictCount);
 329
 330     switch ($this->_contactType) {
 331       case 'Individual':
 332         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 333         break;
 334
 335       case 'Household':
 336         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 337         break;
 338
 339       case 'Organization':
 340         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 341     }
 342
 343     if ($this->_invalidRowCount) {
 344       $store->set('errorsFileName', $this->_errorFileName);
 345     }
 346     if ($this->_conflictCount) {
 347       $store->set('conflictsFileName', $this->_conflictFileName);
 348     }
 349     if (isset($this->_rows) && !empty($this->_rows)) {
 350       $store->set('dataValues', $this->_rows);
 351     }
 352
 353     if ($mode == self::MODE_IMPORT) {
 354       $store->set('duplicateRowCount', $this->_duplicateCount);
 355       if ($this->_duplicateCount) {
 356         $store->set('duplicatesFileName', $this->_duplicateFileName);
 357       }
 358     }
 359   }
 360
 361 }