CRM/Activity/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser {
  18
  19   protected $_fileName;
  20
  21   /**
  22    * Imported file size.
  23    * @var int
  24    */
  25   protected $_fileSize;
  26
  27   /**
  28    * Separator being used.
  29    * @var string
  30    */
  31   protected $_separator;
  32
  33   /**
  34    * Total number of lines in file.
  35    * @var int
  36    */
  37   protected $_lineCount;
  38
  39   /**
  40    * Whether the file has a column header or not.
  41    *
  42    * @var bool
  43    */
  44   protected $_haveColumnHeader;
  45
  46   /**
  47    * @param array $fileName
  48    * @param string $separator
  49    * @param $mapper
  50    * @param bool $skipColumnHeader
  51    * @param int $mode
  52    * @param int $onDuplicate
  53    * @param int $statusID
  54    * @param int $totalRowCount
  55    *
  56    * @return mixed
  57    * @throws Exception
  58    */
  59   public function run(
  60     array $fileName,
  61     $separator,
  62     &$mapper,
  63     $skipColumnHeader = FALSE,
  64     $mode = self::MODE_PREVIEW,
  65     $onDuplicate = self::DUPLICATE_SKIP,
  66     $statusID = NULL,
  67     $totalRowCount = NULL
  68   ) {
  69
  70     $fileName = $fileName['name'];
  71
  72     $this->init();
  73
  74     $this->_haveColumnHeader = $skipColumnHeader;
  75
  76     $this->_separator = $separator;
  77
  78     $fd = fopen($fileName, "r");
  79     if (!$fd) {
  80       return FALSE;
  81     }
  82
  83     $this->_lineCount = $this->_warningCount = 0;
  84     $this->_invalidRowCount = $this->_validCount = 0;
  85     $this->_totalCount = $this->_conflictCount = 0;
  86
  87     $this->_errors = [];
  88     $this->_warnings = [];
  89     $this->_conflicts = [];
  90
  91     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
  92
  93     if ($mode == self::MODE_MAPFIELD) {
  94       $this->_rows = [];
  95     }
  96     else {
  97       $this->_activeFieldCount = count($this->_activeFields);
  98     }
  99     if ($statusID) {
 100       $this->progressImport($statusID);
 101       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 102     }
 103
 104     while (!feof($fd)) {
 105       $this->_lineCount++;
 106
 107       $values = fgetcsv($fd, 8192, $separator);
 108       if (!$values) {
 109         continue;
 110       }
 111
 112       self::encloseScrub($values);
 113
 114       // skip column header if we're not in mapfield mode
 115       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 116         $skipColumnHeader = FALSE;
 117         continue;
 118       }
 119
 120       // Trim whitespace around the values.
 121
 122       $empty = TRUE;
 123       foreach ($values as $k => $v) {
 124         $values[$k] = trim($v, " \t\r\n");
 125       }
 126
 127       if (CRM_Utils_System::isNull($values)) {
 128         continue;
 129       }
 130
 131       $this->_totalCount++;
 132
 133       if ($mode == self::MODE_MAPFIELD) {
 134         $returnCode = $this->mapField($values);
 135       }
 136       elseif ($mode == self::MODE_PREVIEW) {
 137         $returnCode = $this->preview($values);
 138       }
 139       elseif ($mode == self::MODE_SUMMARY) {
 140         $returnCode = $this->summary($values);
 141       }
 142       elseif ($mode == self::MODE_IMPORT) {
 143         $returnCode = $this->import($onDuplicate, $values);
 144         if ($statusID && (($this->_lineCount % 50) == 0)) {
 145           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 146         }
 147       }
 148       else {
 149         $returnCode = self::ERROR;
 150       }
 151
 152       // note that a line could be valid but still produce a warning
 153       if ($returnCode & self::VALID) {
 154         $this->_validCount++;
 155         if ($mode == self::MODE_MAPFIELD) {
 156           $this->_rows[] = $values;
 157           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 158         }
 159       }
 160
 161       if ($returnCode & self::WARNING) {
 162         $this->_warningCount++;
 163         if ($this->_warningCount < $this->_maxWarningCount) {
 164           $this->_warningCount[] = $line;
 165         }
 166       }
 167
 168       if ($returnCode & self::ERROR) {
 169         $this->_invalidRowCount++;
 170         $recordNumber = $this->_lineCount;
 171         if ($this->_haveColumnHeader) {
 172           $recordNumber--;
 173         }
 174         array_unshift($values, $recordNumber);
 175         $this->_errors[] = $values;
 176       }
 177
 178       if ($returnCode & self::CONFLICT) {
 179         $this->_conflictCount++;
 180         $recordNumber = $this->_lineCount;
 181         if ($this->_haveColumnHeader) {
 182           $recordNumber--;
 183         }
 184         array_unshift($values, $recordNumber);
 185         $this->_conflicts[] = $values;
 186       }
 187
 188       if ($returnCode & self::DUPLICATE) {
 189         if ($returnCode & self::MULTIPLE_DUPE) {
 190           // TODO: multi-dupes should be counted apart from singles
 191           // on non-skip action.
 192         }
 193         $this->_duplicateCount++;
 194         $recordNumber = $this->_lineCount;
 195         if ($this->_haveColumnHeader) {
 196           $recordNumber--;
 197         }
 198         array_unshift($values, $recordNumber);
 199         $this->_duplicates[] = $values;
 200         if ($onDuplicate != self::DUPLICATE_SKIP) {
 201           $this->_validCount++;
 202         }
 203       }
 204
 205       // we give the derived class a way of aborting the process
 206       // note that the return code could be multiple code or'ed together
 207       if ($returnCode & self::STOP) {
 208         break;
 209       }
 210
 211       // if we are done processing the maxNumber of lines, break
 212       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 213         break;
 214       }
 215     }
 216
 217     fclose($fd);
 218
 219     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 220       $customHeaders = $mapper;
 221
 222       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 223       foreach ($customHeaders as $key => $value) {
 224         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 225           $customHeaders[$key] = $customfields[$id][0];
 226         }
 227       }
 228       if ($this->_invalidRowCount) {
 229         // removed view url for invlaid contacts
 230         $headers = array_merge(
 231           [ts('Line Number'), ts('Reason')],
 232           $customHeaders
 233         );
 234         $this->_errorFileName = self::errorFileName(self::ERROR);
 235         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 236       }
 237       if ($this->_conflictCount) {
 238         $headers = array_merge(
 239           [ts('Line Number'), ts('Reason')],
 240           $customHeaders
 241         );
 242         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 243         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 244       }
 245       if ($this->_duplicateCount) {
 246         $headers = array_merge(
 247           [ts('Line Number'), ts('View Activity History URL')],
 248           $customHeaders
 249         );
 250
 251         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 252         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 253       }
 254     }
 255     return $this->fini();
 256   }
 257
 258   /**
 259    * Given a list of the importable field keys that the user has selected set the active fields array to this list.
 260    *
 261    * @param array $fieldKeys
 262    */
 263   public function setActiveFields($fieldKeys) {
 264     $this->_activeFieldCount = count($fieldKeys);
 265     foreach ($fieldKeys as $key) {
 266       if (empty($this->_fields[$key])) {
 267         $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
 268       }
 269       else {
 270         $this->_activeFields[] = clone($this->_fields[$key]);
 271       }
 272     }
 273   }
 274
 275   /**
 276    * @param string $name
 277    * @param $title
 278    * @param int $type
 279    * @param string $headerPattern
 280    * @param string $dataPattern
 281    */
 282   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 283     if (empty($name)) {
 284       $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 285     }
 286     else {
 287
 288       $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
 289       if (!array_key_exists($name, $tempField)) {
 290         $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 291       }
 292       else {
 293         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
 294       }
 295     }
 296   }
 297
 298   /**
 299    * Store parser values.
 300    *
 301    * @param CRM_Core_Session $store
 302    *
 303    * @param int $mode
 304    */
 305   public function set($store, $mode = self::MODE_SUMMARY) {
 306     $store->set('fileSize', $this->_fileSize);
 307     $store->set('lineCount', $this->_lineCount);
 308     $store->set('separator', $this->_separator);
 309     $store->set('fields', $this->getSelectValues());
 310
 311     $store->set('headerPatterns', $this->getHeaderPatterns());
 312     $store->set('dataPatterns', $this->getDataPatterns());
 313     $store->set('columnCount', $this->_activeFieldCount);
 314
 315     $store->set('totalRowCount', $this->_totalCount);
 316     $store->set('validRowCount', $this->_validCount);
 317     $store->set('invalidRowCount', $this->_invalidRowCount);
 318     $store->set('conflictRowCount', $this->_conflictCount);
 319
 320     if ($this->_invalidRowCount) {
 321       $store->set('errorsFileName', $this->_errorFileName);
 322     }
 323     if ($this->_conflictCount) {
 324       $store->set('conflictsFileName', $this->_conflictFileName);
 325     }
 326     if (isset($this->_rows) && !empty($this->_rows)) {
 327       $store->set('dataValues', $this->_rows);
 328     }
 329
 330     if ($mode == self::MODE_IMPORT) {
 331       $store->set('duplicateRowCount', $this->_duplicateCount);
 332       if ($this->_duplicateCount) {
 333         $store->set('duplicatesFileName', $this->_duplicateFileName);
 334       }
 335     }
 336   }
 337
 338   /**
 339    * Export data to a CSV file.
 340    *
 341    * @param string $fileName
 342    * @param array $header
 343    * @param array $data
 344    */
 345   public static function exportCSV($fileName, $header, $data) {
 346     $output = [];
 347     $fd = fopen($fileName, 'w');
 348
 349     foreach ($header as $key => $value) {
 350       $header[$key] = "\"$value\"";
 351     }
 352     $config = CRM_Core_Config::singleton();
 353     $output[] = implode($config->fieldSeparator, $header);
 354
 355     foreach ($data as $datum) {
 356       foreach ($datum as $key => $value) {
 357         $datum[$key] = "\"$value\"";
 358       }
 359       $output[] = implode($config->fieldSeparator, $datum);
 360     }
 361     fwrite($fd, implode("\n", $output));
 362     fclose($fd);
 363   }
 364
 365 }