CRM/Activity/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser {
  18
  19   protected $_fileName;
  20
  21   /**
  22    * Imported file size.
  23    * @var int
  24    */
  25   protected $_fileSize;
  26
  27   /**
  28    * Separator being used.
  29    * @var string
  30    */
  31   protected $_seperator;
  32
  33   /**
  34    * Total number of lines in file.
  35    * @var int
  36    */
  37   protected $_lineCount;
  38
  39   /**
  40    * Whether the file has a column header or not.
  41    *
  42    * @var bool
  43    */
  44   protected $_haveColumnHeader;
  45
  46   /**
  47    * @param string $fileName
  48    * @param string $seperator
  49    * @param $mapper
  50    * @param bool $skipColumnHeader
  51    * @param int $mode
  52    * @param int $onDuplicate
  53    * @param int $statusID
  54    * @param int $totalRowCount
  55    *
  56    * @return mixed
  57    * @throws Exception
  58    */
  59   public function run(
  60     $fileName,
  61     $seperator = ',',
  62     &$mapper,
  63     $skipColumnHeader = FALSE,
  64     $mode = self::MODE_PREVIEW,
  65     $onDuplicate = self::DUPLICATE_SKIP,
  66     $statusID = NULL,
  67     $totalRowCount = NULL
  68   ) {
  69     if (!is_array($fileName)) {
  70       CRM_Core_Error::fatal();
  71     }
  72     $fileName = $fileName['name'];
  73
  74     $this->init();
  75
  76     $this->_haveColumnHeader = $skipColumnHeader;
  77
  78     $this->_seperator = $seperator;
  79
  80     $fd = fopen($fileName, "r");
  81     if (!$fd) {
  82       return FALSE;
  83     }
  84
  85     $this->_lineCount = $this->_warningCount = 0;
  86     $this->_invalidRowCount = $this->_validCount = 0;
  87     $this->_totalCount = $this->_conflictCount = 0;
  88
  89     $this->_errors = [];
  90     $this->_warnings = [];
  91     $this->_conflicts = [];
  92
  93     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
  94
  95     if ($mode == self::MODE_MAPFIELD) {
  96       $this->_rows = [];
  97     }
  98     else {
  99       $this->_activeFieldCount = count($this->_activeFields);
 100     }
 101     if ($statusID) {
 102       $this->progressImport($statusID);
 103       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 104     }
 105
 106     while (!feof($fd)) {
 107       $this->_lineCount++;
 108
 109       $values = fgetcsv($fd, 8192, $seperator);
 110       if (!$values) {
 111         continue;
 112       }
 113
 114       self::encloseScrub($values);
 115
 116       // skip column header if we're not in mapfield mode
 117       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 118         $skipColumnHeader = FALSE;
 119         continue;
 120       }
 121
 122       // Trim whitespace around the values.
 123
 124       $empty = TRUE;
 125       foreach ($values as $k => $v) {
 126         $values[$k] = trim($v, " \t\r\n");
 127       }
 128
 129       if (CRM_Utils_System::isNull($values)) {
 130         continue;
 131       }
 132
 133       $this->_totalCount++;
 134
 135       if ($mode == self::MODE_MAPFIELD) {
 136         $returnCode = $this->mapField($values);
 137       }
 138       elseif ($mode == self::MODE_PREVIEW) {
 139         $returnCode = $this->preview($values);
 140       }
 141       elseif ($mode == self::MODE_SUMMARY) {
 142         $returnCode = $this->summary($values);
 143       }
 144       elseif ($mode == self::MODE_IMPORT) {
 145         $returnCode = $this->import($onDuplicate, $values);
 146         if ($statusID && (($this->_lineCount % 50) == 0)) {
 147           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 148         }
 149       }
 150       else {
 151         $returnCode = self::ERROR;
 152       }
 153
 154       // note that a line could be valid but still produce a warning
 155       if ($returnCode & self::VALID) {
 156         $this->_validCount++;
 157         if ($mode == self::MODE_MAPFIELD) {
 158           $this->_rows[] = $values;
 159           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 160         }
 161       }
 162
 163       if ($returnCode & self::WARNING) {
 164         $this->_warningCount++;
 165         if ($this->_warningCount < $this->_maxWarningCount) {
 166           $this->_warningCount[] = $line;
 167         }
 168       }
 169
 170       if ($returnCode & self::ERROR) {
 171         $this->_invalidRowCount++;
 172         $recordNumber = $this->_lineCount;
 173         if ($this->_haveColumnHeader) {
 174           $recordNumber--;
 175         }
 176         array_unshift($values, $recordNumber);
 177         $this->_errors[] = $values;
 178       }
 179
 180       if ($returnCode & self::CONFLICT) {
 181         $this->_conflictCount++;
 182         $recordNumber = $this->_lineCount;
 183         if ($this->_haveColumnHeader) {
 184           $recordNumber--;
 185         }
 186         array_unshift($values, $recordNumber);
 187         $this->_conflicts[] = $values;
 188       }
 189
 190       if ($returnCode & self::DUPLICATE) {
 191         if ($returnCode & self::MULTIPLE_DUPE) {
 192           // TODO: multi-dupes should be counted apart from singles
 193           // on non-skip action.
 194         }
 195         $this->_duplicateCount++;
 196         $recordNumber = $this->_lineCount;
 197         if ($this->_haveColumnHeader) {
 198           $recordNumber--;
 199         }
 200         array_unshift($values, $recordNumber);
 201         $this->_duplicates[] = $values;
 202         if ($onDuplicate != self::DUPLICATE_SKIP) {
 203           $this->_validCount++;
 204         }
 205       }
 206
 207       // we give the derived class a way of aborting the process
 208       // note that the return code could be multiple code or'ed together
 209       if ($returnCode & self::STOP) {
 210         break;
 211       }
 212
 213       // if we are done processing the maxNumber of lines, break
 214       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 215         break;
 216       }
 217     }
 218
 219     fclose($fd);
 220
 221     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 222       $customHeaders = $mapper;
 223
 224       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 225       foreach ($customHeaders as $key => $value) {
 226         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 227           $customHeaders[$key] = $customfields[$id][0];
 228         }
 229       }
 230       if ($this->_invalidRowCount) {
 231         // removed view url for invlaid contacts
 232         $headers = array_merge(
 233           [ts('Line Number'), ts('Reason')],
 234           $customHeaders
 235         );
 236         $this->_errorFileName = self::errorFileName(self::ERROR);
 237         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 238       }
 239       if ($this->_conflictCount) {
 240         $headers = array_merge(
 241           [ts('Line Number'), ts('Reason')],
 242           $customHeaders
 243         );
 244         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 245         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 246       }
 247       if ($this->_duplicateCount) {
 248         $headers = array_merge(
 249           [ts('Line Number'), ts('View Activity History URL')],
 250           $customHeaders
 251         );
 252
 253         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 254         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 255       }
 256     }
 257     return $this->fini();
 258   }
 259
 260   /**
 261    * Given a list of the importable field keys that the user has selected set the active fields array to this list.
 262    *
 263    * @param array $fieldKeys
 264    */
 265   public function setActiveFields($fieldKeys) {
 266     $this->_activeFieldCount = count($fieldKeys);
 267     foreach ($fieldKeys as $key) {
 268       if (empty($this->_fields[$key])) {
 269         $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
 270       }
 271       else {
 272         $this->_activeFields[] = clone($this->_fields[$key]);
 273       }
 274     }
 275   }
 276
 277   /**
 278    * Format the field values for input to the api.
 279    *
 280    * @return array
 281    *   (reference ) associative array of name/value pairs
 282    */
 283   public function &getActiveFieldParams() {
 284     $params = [];
 285     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 286       if (isset($this->_activeFields[$i]->_value)
 287         && !isset($params[$this->_activeFields[$i]->_name])
 288         && !isset($this->_activeFields[$i]->_related)
 289       ) {
 290
 291         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 292       }
 293     }
 294     return $params;
 295   }
 296
 297   /**
 298    * @param string $name
 299    * @param $title
 300    * @param int $type
 301    * @param string $headerPattern
 302    * @param string $dataPattern
 303    */
 304   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 305     if (empty($name)) {
 306       $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 307     }
 308     else {
 309
 310       $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
 311       if (!array_key_exists($name, $tempField)) {
 312         $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 313       }
 314       else {
 315         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
 316       }
 317     }
 318   }
 319
 320   /**
 321    * Store parser values.
 322    *
 323    * @param CRM_Core_Session $store
 324    *
 325    * @param int $mode
 326    */
 327   public function set($store, $mode = self::MODE_SUMMARY) {
 328     $store->set('fileSize', $this->_fileSize);
 329     $store->set('lineCount', $this->_lineCount);
 330     $store->set('seperator', $this->_seperator);
 331     $store->set('fields', $this->getSelectValues());
 332     $store->set('fieldTypes', $this->getSelectTypes());
 333
 334     $store->set('headerPatterns', $this->getHeaderPatterns());
 335     $store->set('dataPatterns', $this->getDataPatterns());
 336     $store->set('columnCount', $this->_activeFieldCount);
 337
 338     $store->set('totalRowCount', $this->_totalCount);
 339     $store->set('validRowCount', $this->_validCount);
 340     $store->set('invalidRowCount', $this->_invalidRowCount);
 341     $store->set('conflictRowCount', $this->_conflictCount);
 342
 343     if ($this->_invalidRowCount) {
 344       $store->set('errorsFileName', $this->_errorFileName);
 345     }
 346     if ($this->_conflictCount) {
 347       $store->set('conflictsFileName', $this->_conflictFileName);
 348     }
 349     if (isset($this->_rows) && !empty($this->_rows)) {
 350       $store->set('dataValues', $this->_rows);
 351     }
 352
 353     if ($mode == self::MODE_IMPORT) {
 354       $store->set('duplicateRowCount', $this->_duplicateCount);
 355       if ($this->_duplicateCount) {
 356         $store->set('duplicatesFileName', $this->_duplicateFileName);
 357       }
 358     }
 359   }
 360
 361   /**
 362    * Export data to a CSV file.
 363    *
 364    * @param string $fileName
 365    * @param array $header
 366    * @param array $data
 367    */
 368   public static function exportCSV($fileName, $header, $data) {
 369     $output = [];
 370     $fd = fopen($fileName, 'w');
 371
 372     foreach ($header as $key => $value) {
 373       $header[$key] = "\"$value\"";
 374     }
 375     $config = CRM_Core_Config::singleton();
 376     $output[] = implode($config->fieldSeparator, $header);
 377
 378     foreach ($data as $datum) {
 379       foreach ($datum as $key => $value) {
 380         $datum[$key] = "\"$value\"";
 381       }
 382       $output[] = implode($config->fieldSeparator, $datum);
 383     }
 384     fwrite($fd, implode("\n", $output));
 385     fclose($fd);
 386   }
 387
 388 }