CRM/Member/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Member_Import_Parser extends CRM_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**
  40    * Imported file size
  41    * @var int
  42    */
  43   protected $_fileSize;
  44
  45   /**
  46    * Seperator being used
  47    * @var string
  48    */
  49   protected $_seperator;
  50
  51   /**
  52    * Total number of lines in file
  53    * @var int
  54    */
  55   protected $_lineCount;
  56
  57   /**
  58    * Whether the file has a column header or not
  59    *
  60    * @var bool
  61    */
  62   protected $_haveColumnHeader;
  63
  64   /**
  65    * @param string $fileName
  66    * @param string $seperator
  67    * @param $mapper
  68    * @param bool $skipColumnHeader
  69    * @param int $mode
  70    * @param int $contactType
  71    * @param int $onDuplicate
  72    * @param int $statusID
  73    * @param int $totalRowCount
  74    *
  75    * @return mixed
  76    * @throws Exception
  77    */
  78   public function run(
  79     $fileName,
  80     $seperator = ',',
  81     &$mapper,
  82     $skipColumnHeader = FALSE,
  83     $mode = self::MODE_PREVIEW,
  84     $contactType = self::CONTACT_INDIVIDUAL,
  85     $onDuplicate = self::DUPLICATE_SKIP,
  86     $statusID = NULL,
  87     $totalRowCount = NULL
  88   ) {
  89     if (!is_array($fileName)) {
  90       CRM_Core_Error::fatal();
  91     }
  92     $fileName = $fileName['name'];
  93
  94     switch ($contactType) {
  95       case self::CONTACT_INDIVIDUAL:
  96         $this->_contactType = 'Individual';
  97         break;
  98
  99       case self::CONTACT_HOUSEHOLD:
 100         $this->_contactType = 'Household';
 101         break;
 102
 103       case self::CONTACT_ORGANIZATION:
 104         $this->_contactType = 'Organization';
 105     }
 106
 107     $this->init();
 108
 109     $this->_haveColumnHeader = $skipColumnHeader;
 110
 111     $this->_seperator = $seperator;
 112
 113     $fd = fopen($fileName, "r");
 114     if (!$fd) {
 115       return FALSE;
 116     }
 117
 118     $this->_lineCount = $this->_warningCount = 0;
 119     $this->_invalidRowCount = $this->_validCount = 0;
 120     $this->_totalCount = $this->_conflictCount = 0;
 121
 122     $this->_errors = [];
 123     $this->_warnings = [];
 124     $this->_conflicts = [];
 125
 126     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 127
 128     if ($mode == self::MODE_MAPFIELD) {
 129       $this->_rows = [];
 130     }
 131     else {
 132       $this->_activeFieldCount = count($this->_activeFields);
 133     }
 134     if ($statusID) {
 135       $this->progressImport($statusID);
 136       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 137     }
 138
 139     while (!feof($fd)) {
 140       $this->_lineCount++;
 141
 142       $values = fgetcsv($fd, 8192, $seperator);
 143       if (!$values) {
 144         continue;
 145       }
 146
 147       self::encloseScrub($values);
 148
 149       // skip column header if we're not in mapfield mode
 150       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 151         $skipColumnHeader = FALSE;
 152         continue;
 153       }
 154
 155       /* trim whitespace around the values */
 156       $empty = TRUE;
 157       foreach ($values as $k => $v) {
 158         $values[$k] = trim($v, " \t\r\n");
 159       }
 160       if (CRM_Utils_System::isNull($values)) {
 161         continue;
 162       }
 163
 164       $this->_totalCount++;
 165
 166       if ($mode == self::MODE_MAPFIELD) {
 167         $returnCode = $this->mapField($values);
 168       }
 169       elseif ($mode == self::MODE_PREVIEW) {
 170         $returnCode = $this->preview($values);
 171       }
 172       elseif ($mode == self::MODE_SUMMARY) {
 173         $returnCode = $this->summary($values);
 174       }
 175       elseif ($mode == self::MODE_IMPORT) {
 176         $returnCode = $this->import($onDuplicate, $values);
 177         if ($statusID && (($this->_lineCount % 50) == 0)) {
 178           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 179         }
 180       }
 181       else {
 182         $returnCode = self::ERROR;
 183       }
 184
 185       // note that a line could be valid but still produce a warning
 186       if ($returnCode & self::VALID) {
 187         $this->_validCount++;
 188         if ($mode == self::MODE_MAPFIELD) {
 189           $this->_rows[] = $values;
 190           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 191         }
 192       }
 193
 194       if ($returnCode & self::WARNING) {
 195         $this->_warningCount++;
 196         if ($this->_warningCount < $this->_maxWarningCount) {
 197           $this->_warningCount[] = $line;
 198         }
 199       }
 200
 201       if ($returnCode & self::ERROR) {
 202         $this->_invalidRowCount++;
 203         $recordNumber = $this->_lineCount;
 204         array_unshift($values, $recordNumber);
 205         $this->_errors[] = $values;
 206       }
 207
 208       if ($returnCode & self::CONFLICT) {
 209         $this->_conflictCount++;
 210         $recordNumber = $this->_lineCount;
 211         array_unshift($values, $recordNumber);
 212         $this->_conflicts[] = $values;
 213       }
 214
 215       if ($returnCode & self::DUPLICATE) {
 216         if ($returnCode & self::MULTIPLE_DUPE) {
 217           /* TODO: multi-dupes should be counted apart from singles
 218            * on non-skip action */
 219         }
 220         $this->_duplicateCount++;
 221         $recordNumber = $this->_lineCount;
 222         array_unshift($values, $recordNumber);
 223         $this->_duplicates[] = $values;
 224         if ($onDuplicate != self::DUPLICATE_SKIP) {
 225           $this->_validCount++;
 226         }
 227       }
 228
 229       // we give the derived class a way of aborting the process
 230       // note that the return code could be multiple code or'ed together
 231       if ($returnCode & self::STOP) {
 232         break;
 233       }
 234
 235       // if we are done processing the maxNumber of lines, break
 236       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 237         break;
 238       }
 239     }
 240
 241     fclose($fd);
 242
 243     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 244       $customHeaders = $mapper;
 245
 246       $customfields = CRM_Core_BAO_CustomField::getFields('Membership');
 247       foreach ($customHeaders as $key => $value) {
 248         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 249           $customHeaders[$key] = $customfields[$id][0];
 250         }
 251       }
 252       if ($this->_invalidRowCount) {
 253         // removed view url for invlaid contacts
 254         $headers = array_merge([
 255           ts('Line Number'),
 256           ts('Reason'),
 257         ], $customHeaders);
 258         $this->_errorFileName = self::errorFileName(self::ERROR);
 259
 260         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 261       }
 262       if ($this->_conflictCount) {
 263         $headers = array_merge([
 264           ts('Line Number'),
 265           ts('Reason'),
 266         ], $customHeaders);
 267         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 268         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 269       }
 270       if ($this->_duplicateCount) {
 271         $headers = array_merge([
 272           ts('Line Number'),
 273           ts('View Membership URL'),
 274         ], $customHeaders);
 275
 276         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 277         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 278       }
 279     }
 280     return $this->fini();
 281   }
 282
 283   /**
 284    * Given a list of the importable field keys that the user has selected
 285    * set the active fields array to this list
 286    *
 287    * @param array $fieldKeys mapped array of values
 288    *
 289    * @return void
 290    */
 291   public function setActiveFields($fieldKeys) {
 292     $this->_activeFieldCount = count($fieldKeys);
 293     foreach ($fieldKeys as $key) {
 294       if (empty($this->_fields[$key])) {
 295         $this->_activeFields[] = new CRM_Member_Import_Field('', ts('- do not import -'));
 296       }
 297       else {
 298         $this->_activeFields[] = clone($this->_fields[$key]);
 299       }
 300     }
 301   }
 302
 303   /**
 304    * Format the field values for input to the api.
 305    *
 306    * @return array
 307    *   (reference ) associative array of name/value pairs
 308    */
 309   public function &getActiveFieldParams() {
 310     $params = [];
 311     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 312       if (isset($this->_activeFields[$i]->_value)
 313         && !isset($params[$this->_activeFields[$i]->_name])
 314         && !isset($this->_activeFields[$i]->_related)
 315       ) {
 316
 317         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 318       }
 319     }
 320     return $params;
 321   }
 322
 323   /**
 324    * @param string $name
 325    * @param $title
 326    * @param int $type
 327    * @param string $headerPattern
 328    * @param string $dataPattern
 329    */
 330   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 331     if (empty($name)) {
 332       $this->_fields['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 333     }
 334     else {
 335
 336       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 337       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 338       if (!array_key_exists($name, $tempField)) {
 339         $this->_fields[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 340       }
 341       else {
 342         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 343           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 344         );
 345       }
 346     }
 347   }
 348
 349   /**
 350    * Store parser values.
 351    *
 352    * @param CRM_Core_Session $store
 353    *
 354    * @param int $mode
 355    *
 356    * @return void
 357    */
 358   public function set($store, $mode = self::MODE_SUMMARY) {
 359     $store->set('fileSize', $this->_fileSize);
 360     $store->set('lineCount', $this->_lineCount);
 361     $store->set('seperator', $this->_seperator);
 362     $store->set('fields', $this->getSelectValues());
 363     $store->set('fieldTypes', $this->getSelectTypes());
 364
 365     $store->set('headerPatterns', $this->getHeaderPatterns());
 366     $store->set('dataPatterns', $this->getDataPatterns());
 367     $store->set('columnCount', $this->_activeFieldCount);
 368
 369     $store->set('totalRowCount', $this->_totalCount);
 370     $store->set('validRowCount', $this->_validCount);
 371     $store->set('invalidRowCount', $this->_invalidRowCount);
 372     $store->set('conflictRowCount', $this->_conflictCount);
 373
 374     switch ($this->_contactType) {
 375       case 'Individual':
 376         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 377         break;
 378
 379       case 'Household':
 380         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 381         break;
 382
 383       case 'Organization':
 384         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 385     }
 386
 387     if ($this->_invalidRowCount) {
 388       $store->set('errorsFileName', $this->_errorFileName);
 389     }
 390     if ($this->_conflictCount) {
 391       $store->set('conflictsFileName', $this->_conflictFileName);
 392     }
 393     if (isset($this->_rows) && !empty($this->_rows)) {
 394       $store->set('dataValues', $this->_rows);
 395     }
 396
 397     if ($mode == self::MODE_IMPORT) {
 398       $store->set('duplicateRowCount', $this->_duplicateCount);
 399       if ($this->_duplicateCount) {
 400         $store->set('duplicatesFileName', $this->_duplicateFileName);
 401       }
 402     }
 403   }
 404
 405   /**
 406    * Export data to a CSV file.
 407    *
 408    * @param string $fileName
 409    * @param array $header
 410    * @param array $data
 411    *
 412    * @return void
 413    */
 414   public static function exportCSV($fileName, $header, $data) {
 415     $output = [];
 416     $fd = fopen($fileName, 'w');
 417
 418     foreach ($header as $key => $value) {
 419       $header[$key] = "\"$value\"";
 420     }
 421     $config = CRM_Core_Config::singleton();
 422     $output[] = implode($config->fieldSeparator, $header);
 423
 424     foreach ($data as $datum) {
 425       foreach ($datum as $key => $value) {
 426         if (is_array($value)) {
 427           foreach ($value[0] as $k1 => $v1) {
 428             if ($k1 == 'location_type_id') {
 429               continue;
 430             }
 431             $datum[$k1] = $v1;
 432           }
 433         }
 434         else {
 435           $datum[$key] = "\"$value\"";
 436         }
 437       }
 438       $output[] = implode($config->fieldSeparator, $datum);
 439     }
 440     fwrite($fd, implode("\n", $output));
 441     fclose($fd);
 442   }
 443
 444 }