CRM/Event/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Event_Import_Parser extends CRM_Import_Parser {
  18
  19   protected $_fileName;
  20
  21   /**
  22    * Imported file size.
  23    *
  24    * @var int
  25    */
  26   protected $_fileSize;
  27
  28   /**
  29    * Separator being used.
  30    *
  31    * @var string
  32    */
  33   protected $_seperator;
  34
  35   /**
  36    * Total number of lines in file.
  37    *
  38    * @var int
  39    */
  40   protected $_lineCount;
  41
  42   /**
  43    * Whether the file has a column header or not
  44    *
  45    * @var bool
  46    */
  47   protected $_haveColumnHeader;
  48
  49   /**
  50    * @param string $fileName
  51    * @param string $seperator
  52    * @param $mapper
  53    * @param bool $skipColumnHeader
  54    * @param int $mode
  55    * @param int $contactType
  56    * @param int $onDuplicate
  57    *
  58    * @return mixed
  59    * @throws Exception
  60    */
  61   public function run(
  62     $fileName,
  63     $seperator = ',',
  64     &$mapper,
  65     $skipColumnHeader = FALSE,
  66     $mode = self::MODE_PREVIEW,
  67     $contactType = self::CONTACT_INDIVIDUAL,
  68     $onDuplicate = self::DUPLICATE_SKIP
  69   ) {
  70     if (!is_array($fileName)) {
  71       throw new CRM_Core_Exception('Unable to determine import file');
  72     }
  73     $fileName = $fileName['name'];
  74
  75     switch ($contactType) {
  76       case self::CONTACT_INDIVIDUAL:
  77         $this->_contactType = 'Individual';
  78         break;
  79
  80       case self::CONTACT_HOUSEHOLD:
  81         $this->_contactType = 'Household';
  82         break;
  83
  84       case self::CONTACT_ORGANIZATION:
  85         $this->_contactType = 'Organization';
  86     }
  87
  88     $this->init();
  89
  90     $this->_haveColumnHeader = $skipColumnHeader;
  91
  92     $this->_seperator = $seperator;
  93
  94     $fd = fopen($fileName, "r");
  95     if (!$fd) {
  96       return FALSE;
  97     }
  98
  99     $this->_lineCount = $this->_warningCount = 0;
 100     $this->_invalidRowCount = $this->_validCount = 0;
 101     $this->_totalCount = $this->_conflictCount = 0;
 102
 103     $this->_errors = [];
 104     $this->_warnings = [];
 105     $this->_conflicts = [];
 106
 107     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 108
 109     if ($mode == self::MODE_MAPFIELD) {
 110       $this->_rows = [];
 111     }
 112     else {
 113       $this->_activeFieldCount = count($this->_activeFields);
 114     }
 115
 116     while (!feof($fd)) {
 117       $this->_lineCount++;
 118
 119       $values = fgetcsv($fd, 8192, $seperator);
 120       if (!$values) {
 121         continue;
 122       }
 123
 124       self::encloseScrub($values);
 125
 126       // skip column header if we're not in mapfield mode
 127       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 128         $skipColumnHeader = FALSE;
 129         continue;
 130       }
 131
 132       /* trim whitespace around the values */
 133
 134       $empty = TRUE;
 135       foreach ($values as $k => $v) {
 136         $values[$k] = trim($v, " \t\r\n");
 137       }
 138
 139       if (CRM_Utils_System::isNull($values)) {
 140         continue;
 141       }
 142
 143       $this->_totalCount++;
 144
 145       if ($mode == self::MODE_MAPFIELD) {
 146         $returnCode = $this->mapField($values);
 147       }
 148       elseif ($mode == self::MODE_PREVIEW) {
 149         $returnCode = $this->preview($values);
 150       }
 151       elseif ($mode == self::MODE_SUMMARY) {
 152         $returnCode = $this->summary($values);
 153       }
 154       elseif ($mode == self::MODE_IMPORT) {
 155         $returnCode = $this->import($onDuplicate, $values);
 156       }
 157       else {
 158         $returnCode = self::ERROR;
 159       }
 160
 161       // note that a line could be valid but still produce a warning
 162       if ($returnCode & self::VALID) {
 163         $this->_validCount++;
 164         if ($mode == self::MODE_MAPFIELD) {
 165           $this->_rows[] = $values;
 166           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 167         }
 168       }
 169
 170       if ($returnCode & self::WARNING) {
 171         $this->_warningCount++;
 172         if ($this->_warningCount < $this->_maxWarningCount) {
 173           $this->_warningCount[] = $line;
 174         }
 175       }
 176
 177       if ($returnCode & self::ERROR) {
 178         $this->_invalidRowCount++;
 179         $recordNumber = $this->_lineCount;
 180         if ($this->_haveColumnHeader) {
 181           $recordNumber--;
 182         }
 183         array_unshift($values, $recordNumber);
 184         $this->_errors[] = $values;
 185       }
 186
 187       if ($returnCode & self::CONFLICT) {
 188         $this->_conflictCount++;
 189         $recordNumber = $this->_lineCount;
 190         if ($this->_haveColumnHeader) {
 191           $recordNumber--;
 192         }
 193         array_unshift($values, $recordNumber);
 194         $this->_conflicts[] = $values;
 195       }
 196
 197       if ($returnCode & self::DUPLICATE) {
 198         if ($returnCode & self::MULTIPLE_DUPE) {
 199           /* TODO: multi-dupes should be counted apart from singles
 200            * on non-skip action */
 201         }
 202         $this->_duplicateCount++;
 203         $recordNumber = $this->_lineCount;
 204         if ($this->_haveColumnHeader) {
 205           $recordNumber--;
 206         }
 207         array_unshift($values, $recordNumber);
 208         $this->_duplicates[] = $values;
 209         if ($onDuplicate != self::DUPLICATE_SKIP) {
 210           $this->_validCount++;
 211         }
 212       }
 213
 214       // we give the derived class a way of aborting the process
 215       // note that the return code could be multiple code or'ed together
 216       if ($returnCode & self::STOP) {
 217         break;
 218       }
 219
 220       // if we are done processing the maxNumber of lines, break
 221       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 222         break;
 223       }
 224     }
 225
 226     fclose($fd);
 227
 228     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 229       $customHeaders = $mapper;
 230
 231       $customfields = CRM_Core_BAO_CustomField::getFields('Participant');
 232       foreach ($customHeaders as $key => $value) {
 233         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 234           $customHeaders[$key] = $customfields[$id][0];
 235         }
 236       }
 237
 238       if ($this->_invalidRowCount) {
 239         // removed view url for invlaid contacts
 240         $headers = array_merge([
 241           ts('Line Number'),
 242           ts('Reason'),
 243         ], $customHeaders);
 244         $this->_errorFileName = self::errorFileName(self::ERROR);
 245         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 246       }
 247       if ($this->_conflictCount) {
 248         $headers = array_merge([
 249           ts('Line Number'),
 250           ts('Reason'),
 251         ], $customHeaders);
 252         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 253         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 254       }
 255       if ($this->_duplicateCount) {
 256         $headers = array_merge([
 257           ts('Line Number'),
 258           ts('View Participant URL'),
 259         ], $customHeaders);
 260
 261         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 262         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 263       }
 264     }
 265     return $this->fini();
 266   }
 267
 268   /**
 269    * Given a list of the importable field keys that the user has selected
 270    * set the active fields array to this list
 271    *
 272    * @param $fieldKeys array mapped array of values
 273    *
 274    * @return void
 275    */
 276   public function setActiveFields($fieldKeys) {
 277     $this->_activeFieldCount = count($fieldKeys);
 278     foreach ($fieldKeys as $key) {
 279       if (empty($this->_fields[$key])) {
 280         $this->_activeFields[] = new CRM_Event_Import_Field('', ts('- do not import -'));
 281       }
 282       else {
 283         $this->_activeFields[] = clone($this->_fields[$key]);
 284       }
 285     }
 286   }
 287
 288   /**
 289    * Format the field values for input to the api.
 290    *
 291    * @return array
 292    *   (reference ) associative array of name/value pairs
 293    */
 294   public function &getActiveFieldParams() {
 295     $params = [];
 296     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 297       if (isset($this->_activeFields[$i]->_value)
 298         && !isset($params[$this->_activeFields[$i]->_name])
 299         && !isset($this->_activeFields[$i]->_related)
 300       ) {
 301
 302         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 303       }
 304     }
 305     return $params;
 306   }
 307
 308   /**
 309    * @param string $name
 310    * @param $title
 311    * @param int $type
 312    * @param string $headerPattern
 313    * @param string $dataPattern
 314    */
 315   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 316     if (empty($name)) {
 317       $this->_fields['doNotImport'] = new CRM_Event_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 318     }
 319     else {
 320
 321       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 322       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 323       if (!array_key_exists($name, $tempField)) {
 324         $this->_fields[$name] = new CRM_Event_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 325       }
 326       else {
 327         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 328           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 329         );
 330       }
 331     }
 332   }
 333
 334   /**
 335    * Store parser values.
 336    *
 337    * @param CRM_Core_Session $store
 338    *
 339    * @param int $mode
 340    *
 341    * @return void
 342    */
 343   public function set($store, $mode = self::MODE_SUMMARY) {
 344     $store->set('fileSize', $this->_fileSize);
 345     $store->set('lineCount', $this->_lineCount);
 346     $store->set('seperator', $this->_seperator);
 347     $store->set('fields', $this->getSelectValues());
 348     $store->set('fieldTypes', $this->getSelectTypes());
 349
 350     $store->set('headerPatterns', $this->getHeaderPatterns());
 351     $store->set('dataPatterns', $this->getDataPatterns());
 352     $store->set('columnCount', $this->_activeFieldCount);
 353
 354     $store->set('totalRowCount', $this->_totalCount);
 355     $store->set('validRowCount', $this->_validCount);
 356     $store->set('invalidRowCount', $this->_invalidRowCount);
 357     $store->set('conflictRowCount', $this->_conflictCount);
 358
 359     switch ($this->_contactType) {
 360       case 'Individual':
 361         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 362         break;
 363
 364       case 'Household':
 365         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 366         break;
 367
 368       case 'Organization':
 369         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 370     }
 371
 372     if ($this->_invalidRowCount) {
 373       $store->set('errorsFileName', $this->_errorFileName);
 374     }
 375     if ($this->_conflictCount) {
 376       $store->set('conflictsFileName', $this->_conflictFileName);
 377     }
 378     if (isset($this->_rows) && !empty($this->_rows)) {
 379       $store->set('dataValues', $this->_rows);
 380     }
 381
 382     if ($mode == self::MODE_IMPORT) {
 383       $store->set('duplicateRowCount', $this->_duplicateCount);
 384       if ($this->_duplicateCount) {
 385         $store->set('duplicatesFileName', $this->_duplicateFileName);
 386       }
 387     }
 388   }
 389
 390   /**
 391    * Export data to a CSV file.
 392    *
 393    * @param string $fileName
 394    * @param array $header
 395    * @param array $data
 396    *
 397    * @return void
 398    */
 399   public static function exportCSV($fileName, $header, $data) {
 400     $output = [];
 401     $fd = fopen($fileName, 'w');
 402
 403     foreach ($header as $key => $value) {
 404       $header[$key] = "\"$value\"";
 405     }
 406     $config = CRM_Core_Config::singleton();
 407     $output[] = implode($config->fieldSeparator, $header);
 408
 409     foreach ($data as $datum) {
 410       foreach ($datum as $key => $value) {
 411         if (is_array($value)) {
 412           foreach ($value[0] as $k1 => $v1) {
 413             if ($k1 == 'location_type_id') {
 414               continue;
 415             }
 416             $datum[$k1] = $v1;
 417           }
 418         }
 419         else {
 420           $datum[$key] = "\"$value\"";
 421         }
 422       }
 423       $output[] = implode($config->fieldSeparator, $datum);
 424     }
 425     fwrite($fd, implode("\n", $output));
 426     fclose($fd);
 427   }
 428
 429 }