CRM/Member/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  *
  17  * Class CRM_Member_Import_Parser
  18  */
  19 abstract class CRM_Member_Import_Parser extends CRM_Import_Parser {
  20
  21   protected $_fileName;
  22
  23   /**
  24    * Imported file size
  25    * @var int
  26    */
  27   protected $_fileSize;
  28
  29   /**
  30    * Separator being used
  31    * @var string
  32    */
  33   protected $_separator;
  34
  35   /**
  36    * Total number of lines in file
  37    * @var int
  38    */
  39   protected $_lineCount;
  40
  41   /**
  42    * Whether the file has a column header or not
  43    *
  44    * @var bool
  45    */
  46   protected $_haveColumnHeader;
  47
  48   /**
  49    * @param string $fileName
  50    * @param string $separator
  51    * @param $mapper
  52    * @param bool $skipColumnHeader
  53    * @param int $mode
  54    * @param int $contactType
  55    * @param int $onDuplicate
  56    * @param int $statusID
  57    * @param int $totalRowCount
  58    *
  59    * @return mixed
  60    * @throws Exception
  61    */
  62   public function run(
  63     $fileName,
  64     $separator = ',',
  65     &$mapper,
  66     $skipColumnHeader = FALSE,
  67     $mode = self::MODE_PREVIEW,
  68     $contactType = self::CONTACT_INDIVIDUAL,
  69     $onDuplicate = self::DUPLICATE_SKIP,
  70     $statusID = NULL,
  71     $totalRowCount = NULL
  72   ) {
  73     if (!is_array($fileName)) {
  74       throw new CRM_Core_Exception('Unable to determine import file');
  75     }
  76     $fileName = $fileName['name'];
  77
  78     switch ($contactType) {
  79       case self::CONTACT_INDIVIDUAL:
  80         $this->_contactType = 'Individual';
  81         break;
  82
  83       case self::CONTACT_HOUSEHOLD:
  84         $this->_contactType = 'Household';
  85         break;
  86
  87       case self::CONTACT_ORGANIZATION:
  88         $this->_contactType = 'Organization';
  89     }
  90
  91     $this->init();
  92
  93     $this->_haveColumnHeader = $skipColumnHeader;
  94
  95     $this->_separator = $separator;
  96
  97     $fd = fopen($fileName, "r");
  98     if (!$fd) {
  99       return FALSE;
 100     }
 101
 102     $this->_lineCount = $this->_warningCount = 0;
 103     $this->_invalidRowCount = $this->_validCount = 0;
 104     $this->_totalCount = $this->_conflictCount = 0;
 105
 106     $this->_errors = [];
 107     $this->_warnings = [];
 108     $this->_conflicts = [];
 109
 110     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 111
 112     if ($mode == self::MODE_MAPFIELD) {
 113       $this->_rows = [];
 114     }
 115     else {
 116       $this->_activeFieldCount = count($this->_activeFields);
 117     }
 118     if ($statusID) {
 119       $this->progressImport($statusID);
 120       $startTimestamp = $currTimestamp = $prevTimestamp = CRM_Utils_Time::time();
 121     }
 122
 123     while (!feof($fd)) {
 124       $this->_lineCount++;
 125
 126       $values = fgetcsv($fd, 8192, $separator);
 127       if (!$values) {
 128         continue;
 129       }
 130
 131       self::encloseScrub($values);
 132
 133       // skip column header if we're not in mapfield mode
 134       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 135         $skipColumnHeader = FALSE;
 136         continue;
 137       }
 138
 139       /* trim whitespace around the values */
 140       $empty = TRUE;
 141       foreach ($values as $k => $v) {
 142         $values[$k] = trim($v, " \t\r\n");
 143       }
 144       if (CRM_Utils_System::isNull($values)) {
 145         continue;
 146       }
 147
 148       $this->_totalCount++;
 149
 150       if ($mode == self::MODE_MAPFIELD) {
 151         $returnCode = $this->mapField($values);
 152       }
 153       elseif ($mode == self::MODE_PREVIEW) {
 154         $returnCode = $this->preview($values);
 155       }
 156       elseif ($mode == self::MODE_SUMMARY) {
 157         $returnCode = $this->summary($values);
 158       }
 159       elseif ($mode == self::MODE_IMPORT) {
 160         $returnCode = $this->import($onDuplicate, $values);
 161         if ($statusID && (($this->_lineCount % 50) == 0)) {
 162           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 163         }
 164       }
 165       else {
 166         $returnCode = self::ERROR;
 167       }
 168
 169       // note that a line could be valid but still produce a warning
 170       if ($returnCode & self::VALID) {
 171         $this->_validCount++;
 172         if ($mode == self::MODE_MAPFIELD) {
 173           $this->_rows[] = $values;
 174           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 175         }
 176       }
 177
 178       if ($returnCode & self::WARNING) {
 179         $this->_warningCount++;
 180         if ($this->_warningCount < $this->_maxWarningCount) {
 181           $this->_warningCount[] = $line;
 182         }
 183       }
 184
 185       if ($returnCode & self::ERROR) {
 186         $this->_invalidRowCount++;
 187         $recordNumber = $this->_lineCount;
 188         array_unshift($values, $recordNumber);
 189         $this->_errors[] = $values;
 190       }
 191
 192       if ($returnCode & self::CONFLICT) {
 193         $this->_conflictCount++;
 194         $recordNumber = $this->_lineCount;
 195         array_unshift($values, $recordNumber);
 196         $this->_conflicts[] = $values;
 197       }
 198
 199       if ($returnCode & self::DUPLICATE) {
 200         if ($returnCode & self::MULTIPLE_DUPE) {
 201           /* TODO: multi-dupes should be counted apart from singles
 202            * on non-skip action */
 203         }
 204         $this->_duplicateCount++;
 205         $recordNumber = $this->_lineCount;
 206         array_unshift($values, $recordNumber);
 207         $this->_duplicates[] = $values;
 208         if ($onDuplicate != self::DUPLICATE_SKIP) {
 209           $this->_validCount++;
 210         }
 211       }
 212
 213       // we give the derived class a way of aborting the process
 214       // note that the return code could be multiple code or'ed together
 215       if ($returnCode & self::STOP) {
 216         break;
 217       }
 218
 219       // if we are done processing the maxNumber of lines, break
 220       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 221         break;
 222       }
 223     }
 224
 225     fclose($fd);
 226
 227     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 228       $customHeaders = $mapper;
 229
 230       $customfields = CRM_Core_BAO_CustomField::getFields('Membership');
 231       foreach ($customHeaders as $key => $value) {
 232         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 233           $customHeaders[$key] = $customfields[$id][0];
 234         }
 235       }
 236       if ($this->_invalidRowCount) {
 237         // removed view url for invlaid contacts
 238         $headers = array_merge([
 239           ts('Line Number'),
 240           ts('Reason'),
 241         ], $customHeaders);
 242         $this->_errorFileName = self::errorFileName(self::ERROR);
 243
 244         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 245       }
 246       if ($this->_conflictCount) {
 247         $headers = array_merge([
 248           ts('Line Number'),
 249           ts('Reason'),
 250         ], $customHeaders);
 251         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 252         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 253       }
 254       if ($this->_duplicateCount) {
 255         $headers = array_merge([
 256           ts('Line Number'),
 257           ts('View Membership URL'),
 258         ], $customHeaders);
 259
 260         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 261         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 262       }
 263     }
 264     return $this->fini();
 265   }
 266
 267   /**
 268    * Given a list of the importable field keys that the user has selected
 269    * set the active fields array to this list
 270    *
 271    * @param array $fieldKeys mapped array of values
 272    *
 273    * @return void
 274    */
 275   public function setActiveFields($fieldKeys) {
 276     $this->_activeFieldCount = count($fieldKeys);
 277     foreach ($fieldKeys as $key) {
 278       if (empty($this->_fields[$key])) {
 279         $this->_activeFields[] = new CRM_Member_Import_Field('', ts('- do not import -'));
 280       }
 281       else {
 282         $this->_activeFields[] = clone($this->_fields[$key]);
 283       }
 284     }
 285   }
 286
 287   /**
 288    * Format the field values for input to the api.
 289    *
 290    * @return array
 291    *   (reference ) associative array of name/value pairs
 292    */
 293   public function &getActiveFieldParams() {
 294     $params = [];
 295     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 296       if (isset($this->_activeFields[$i]->_value)
 297         && !isset($params[$this->_activeFields[$i]->_name])
 298         && !isset($this->_activeFields[$i]->_related)
 299       ) {
 300
 301         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 302       }
 303     }
 304     return $params;
 305   }
 306
 307   /**
 308    * @param string $name
 309    * @param $title
 310    * @param int $type
 311    * @param string $headerPattern
 312    * @param string $dataPattern
 313    */
 314   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 315     if (empty($name)) {
 316       $this->_fields['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 317     }
 318     else {
 319
 320       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 321       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 322       if (!array_key_exists($name, $tempField)) {
 323         $this->_fields[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 324       }
 325       else {
 326         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 327           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 328         );
 329       }
 330     }
 331   }
 332
 333   /**
 334    * Store parser values.
 335    *
 336    * @param CRM_Core_Session $store
 337    *
 338    * @param int $mode
 339    *
 340    * @return void
 341    */
 342   public function set($store, $mode = self::MODE_SUMMARY) {
 343     $store->set('fileSize', $this->_fileSize);
 344     $store->set('lineCount', $this->_lineCount);
 345     $store->set('separator', $this->_separator);
 346     $store->set('fields', $this->getSelectValues());
 347     $store->set('fieldTypes', $this->getSelectTypes());
 348
 349     $store->set('headerPatterns', $this->getHeaderPatterns());
 350     $store->set('dataPatterns', $this->getDataPatterns());
 351     $store->set('columnCount', $this->_activeFieldCount);
 352
 353     $store->set('totalRowCount', $this->_totalCount);
 354     $store->set('validRowCount', $this->_validCount);
 355     $store->set('invalidRowCount', $this->_invalidRowCount);
 356     $store->set('conflictRowCount', $this->_conflictCount);
 357
 358     switch ($this->_contactType) {
 359       case 'Individual':
 360         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 361         break;
 362
 363       case 'Household':
 364         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 365         break;
 366
 367       case 'Organization':
 368         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 369     }
 370
 371     if ($this->_invalidRowCount) {
 372       $store->set('errorsFileName', $this->_errorFileName);
 373     }
 374     if ($this->_conflictCount) {
 375       $store->set('conflictsFileName', $this->_conflictFileName);
 376     }
 377     if (isset($this->_rows) && !empty($this->_rows)) {
 378       $store->set('dataValues', $this->_rows);
 379     }
 380
 381     if ($mode == self::MODE_IMPORT) {
 382       $store->set('duplicateRowCount', $this->_duplicateCount);
 383       if ($this->_duplicateCount) {
 384         $store->set('duplicatesFileName', $this->_duplicateFileName);
 385       }
 386     }
 387   }
 388
 389   /**
 390    * Export data to a CSV file.
 391    *
 392    * @param string $fileName
 393    * @param array $header
 394    * @param array $data
 395    *
 396    * @return void
 397    */
 398   public static function exportCSV($fileName, $header, $data) {
 399     $output = [];
 400     $fd = fopen($fileName, 'w');
 401
 402     foreach ($header as $key => $value) {
 403       $header[$key] = "\"$value\"";
 404     }
 405     $config = CRM_Core_Config::singleton();
 406     $output[] = implode($config->fieldSeparator, $header);
 407
 408     foreach ($data as $datum) {
 409       foreach ($datum as $key => $value) {
 410         if (is_array($value)) {
 411           foreach ($value[0] as $k1 => $v1) {
 412             if ($k1 == 'location_type_id') {
 413               continue;
 414             }
 415             $datum[$k1] = $v1;
 416           }
 417         }
 418         else {
 419           $datum[$key] = "\"$value\"";
 420         }
 421       }
 422       $output[] = implode($config->fieldSeparator, $datum);
 423     }
 424     fwrite($fd, implode("\n", $output));
 425     fclose($fd);
 426   }
 427
 428 }