CRM/Activity/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33 abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser {
  34
  35   protected $_fileName;
  36
  37   /**
  38    * Imported file size.
  39    */
  40   protected $_fileSize;
  41
  42   /**
  43    * Separator being used.
  44    */
  45   protected $_seperator;
  46
  47   /**
  48    * Total number of lines in file.
  49    */
  50   protected $_lineCount;
  51
  52   /**
  53    * Whether the file has a column header or not.
  54    *
  55    * @var boolean
  56    */
  57   protected $_haveColumnHeader;
  58
  59   /**
  60    * @param string $fileName
  61    * @param string $seperator
  62    * @param $mapper
  63    * @param bool $skipColumnHeader
  64    * @param int $mode
  65    * @param int $onDuplicate
  66    *
  67    * @return mixed
  68    * @throws Exception
  69    */
  70   public function run(
  71     $fileName,
  72     $seperator = ',',
  73     &$mapper,
  74     $skipColumnHeader = FALSE,
  75     $mode = self::MODE_PREVIEW,
  76     $onDuplicate = self::DUPLICATE_SKIP,
  77     $statusID = NULL,
  78     $totalRowCount = NULL
  79   ) {
  80     if (!is_array($fileName)) {
  81       CRM_Core_Error::fatal();
  82     }
  83     $fileName = $fileName['name'];
  84
  85     $this->init();
  86
  87     $this->_haveColumnHeader = $skipColumnHeader;
  88
  89     $this->_seperator = $seperator;
  90
  91     $fd = fopen($fileName, "r");
  92     if (!$fd) {
  93       return FALSE;
  94     }
  95
  96     $this->_lineCount = $this->_warningCount = 0;
  97     $this->_invalidRowCount = $this->_validCount = 0;
  98     $this->_totalCount = $this->_conflictCount = 0;
  99
 100     $this->_errors = array();
 101     $this->_warnings = array();
 102     $this->_conflicts = array();
 103
 104     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 105
 106     if ($mode == self::MODE_MAPFIELD) {
 107       $this->_rows = array();
 108     }
 109     else {
 110       $this->_activeFieldCount = count($this->_activeFields);
 111     }
 112     if ($statusID) {
 113       $this->progressImport($statusID);
 114       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 115     }
 116
 117     while (!feof($fd)) {
 118       $this->_lineCount++;
 119
 120       $values = fgetcsv($fd, 8192, $seperator);
 121       if (!$values) {
 122         continue;
 123       }
 124
 125       self::encloseScrub($values);
 126
 127       // skip column header if we're not in mapfield mode
 128       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 129         $skipColumnHeader = FALSE;
 130         continue;
 131       }
 132
 133       // Trim whitespace around the values.
 134
 135       $empty = TRUE;
 136       foreach ($values as $k => $v) {
 137         $values[$k] = trim($v, " \t\r\n");
 138       }
 139
 140       if (CRM_Utils_System::isNull($values)) {
 141         continue;
 142       }
 143
 144       $this->_totalCount++;
 145
 146       if ($mode == self::MODE_MAPFIELD) {
 147         $returnCode = $this->mapField($values);
 148       }
 149       elseif ($mode == self::MODE_PREVIEW) {
 150         $returnCode = $this->preview($values);
 151       }
 152       elseif ($mode == self::MODE_SUMMARY) {
 153         $returnCode = $this->summary($values);
 154       }
 155       elseif ($mode == self::MODE_IMPORT) {
 156         $returnCode = $this->import($onDuplicate, $values);
 157         if ($statusID && (($this->_lineCount % 50) == 0)) {
 158           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 159         }
 160       }
 161       else {
 162         $returnCode = self::ERROR;
 163       }
 164
 165       // note that a line could be valid but still produce a warning
 166       if ($returnCode & self::VALID) {
 167         $this->_validCount++;
 168         if ($mode == self::MODE_MAPFIELD) {
 169           $this->_rows[] = $values;
 170           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 171         }
 172       }
 173
 174       if ($returnCode & self::WARNING) {
 175         $this->_warningCount++;
 176         if ($this->_warningCount < $this->_maxWarningCount) {
 177           $this->_warningCount[] = $line;
 178         }
 179       }
 180
 181       if ($returnCode & self::ERROR) {
 182         $this->_invalidRowCount++;
 183         $recordNumber = $this->_lineCount;
 184         if ($this->_haveColumnHeader) {
 185           $recordNumber--;
 186         }
 187         array_unshift($values, $recordNumber);
 188         $this->_errors[] = $values;
 189       }
 190
 191       if ($returnCode & self::CONFLICT) {
 192         $this->_conflictCount++;
 193         $recordNumber = $this->_lineCount;
 194         if ($this->_haveColumnHeader) {
 195           $recordNumber--;
 196         }
 197         array_unshift($values, $recordNumber);
 198         $this->_conflicts[] = $values;
 199       }
 200
 201       if ($returnCode & self::DUPLICATE) {
 202         if ($returnCode & self::MULTIPLE_DUPE) {
 203           // TODO: multi-dupes should be counted apart from singles
 204           // on non-skip action.
 205         }
 206         $this->_duplicateCount++;
 207         $recordNumber = $this->_lineCount;
 208         if ($this->_haveColumnHeader) {
 209           $recordNumber--;
 210         }
 211         array_unshift($values, $recordNumber);
 212         $this->_duplicates[] = $values;
 213         if ($onDuplicate != self::DUPLICATE_SKIP) {
 214           $this->_validCount++;
 215         }
 216       }
 217
 218       // we give the derived class a way of aborting the process
 219       // note that the return code could be multiple code or'ed together
 220       if ($returnCode & self::STOP) {
 221         break;
 222       }
 223
 224       // if we are done processing the maxNumber of lines, break
 225       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 226         break;
 227       }
 228     }
 229
 230     fclose($fd);
 231
 232     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 233       $customHeaders = $mapper;
 234
 235       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 236       foreach ($customHeaders as $key => $value) {
 237         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 238           $customHeaders[$key] = $customfields[$id][0];
 239         }
 240       }
 241       if ($this->_invalidRowCount) {
 242         // removed view url for invlaid contacts
 243         $headers = array_merge(array(
 244             ts('Line Number'),
 245             ts('Reason'),
 246           ),
 247           $customHeaders
 248         );
 249         $this->_errorFileName = self::errorFileName(self::ERROR);
 250         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 251       }
 252       if ($this->_conflictCount) {
 253         $headers = array_merge(array(
 254             ts('Line Number'),
 255             ts('Reason'),
 256           ),
 257           $customHeaders
 258         );
 259         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 260         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 261       }
 262       if ($this->_duplicateCount) {
 263         $headers = array_merge(array(
 264             ts('Line Number'),
 265             ts('View Activity History URL'),
 266           ),
 267           $customHeaders
 268         );
 269
 270         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 271         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 272       }
 273     }
 274     return $this->fini();
 275   }
 276
 277   /**
 278    * Given a list of the importable field keys that the user has selected set the active fields array to this list.
 279    *
 280    * @param array $fieldKeys
 281    */
 282   public function setActiveFields($fieldKeys) {
 283     $this->_activeFieldCount = count($fieldKeys);
 284     foreach ($fieldKeys as $key) {
 285       if (empty($this->_fields[$key])) {
 286         $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
 287       }
 288       else {
 289         $this->_activeFields[] = clone($this->_fields[$key]);
 290       }
 291     }
 292   }
 293
 294   /**
 295    * Format the field values for input to the api.
 296    *
 297    * @return array
 298    *   (reference ) associative array of name/value pairs
 299    */
 300   public function &getActiveFieldParams() {
 301     $params = array();
 302     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 303       if (isset($this->_activeFields[$i]->_value)
 304         && !isset($params[$this->_activeFields[$i]->_name])
 305         && !isset($this->_activeFields[$i]->_related)
 306       ) {
 307
 308         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 309       }
 310     }
 311     return $params;
 312   }
 313
 314   /**
 315    * @param string $name
 316    * @param $title
 317    * @param int $type
 318    * @param string $headerPattern
 319    * @param string $dataPattern
 320    */
 321   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 322     if (empty($name)) {
 323       $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 324     }
 325     else {
 326
 327       $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
 328       if (!array_key_exists($name, $tempField)) {
 329         $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 330       }
 331       else {
 332         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
 333       }
 334     }
 335   }
 336
 337   /**
 338    * Store parser values.
 339    *
 340    * @param CRM_Core_Session $store
 341    *
 342    * @param int $mode
 343    */
 344   public function set($store, $mode = self::MODE_SUMMARY) {
 345     $store->set('fileSize', $this->_fileSize);
 346     $store->set('lineCount', $this->_lineCount);
 347     $store->set('seperator', $this->_seperator);
 348     $store->set('fields', $this->getSelectValues());
 349     $store->set('fieldTypes', $this->getSelectTypes());
 350
 351     $store->set('headerPatterns', $this->getHeaderPatterns());
 352     $store->set('dataPatterns', $this->getDataPatterns());
 353     $store->set('columnCount', $this->_activeFieldCount);
 354
 355     $store->set('totalRowCount', $this->_totalCount);
 356     $store->set('validRowCount', $this->_validCount);
 357     $store->set('invalidRowCount', $this->_invalidRowCount);
 358     $store->set('conflictRowCount', $this->_conflictCount);
 359
 360     if ($this->_invalidRowCount) {
 361       $store->set('errorsFileName', $this->_errorFileName);
 362     }
 363     if ($this->_conflictCount) {
 364       $store->set('conflictsFileName', $this->_conflictFileName);
 365     }
 366     if (isset($this->_rows) && !empty($this->_rows)) {
 367       $store->set('dataValues', $this->_rows);
 368     }
 369
 370     if ($mode == self::MODE_IMPORT) {
 371       $store->set('duplicateRowCount', $this->_duplicateCount);
 372       if ($this->_duplicateCount) {
 373         $store->set('duplicatesFileName', $this->_duplicateFileName);
 374       }
 375     }
 376   }
 377
 378   /**
 379    * Export data to a CSV file.
 380    *
 381    * @param string $fileName
 382    * @param array $header
 383    * @param array $data
 384    */
 385   public static function exportCSV($fileName, $header, $data) {
 386     $output = array();
 387     $fd = fopen($fileName, 'w');
 388
 389     foreach ($header as $key => $value) {
 390       $header[$key] = "\"$value\"";
 391     }
 392     $config = CRM_Core_Config::singleton();
 393     $output[] = implode($config->fieldSeparator, $header);
 394
 395     foreach ($data as $datum) {
 396       foreach ($datum as $key => $value) {
 397         $datum[$key] = "\"$value\"";
 398       }
 399       $output[] = implode($config->fieldSeparator, $datum);
 400     }
 401     fwrite($fd, implode("\n", $output));
 402     fclose($fd);
 403   }
 404
 405 }