CRM/Member/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Member_Import_Parser extends CRM_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**
  40    * #@+
  41    * @var integer
  42    */
  43
  44   /**
  45    * Imported file size
  46    * @var int
  47    */
  48   protected $_fileSize;
  49
  50   /**
  51    * Seperator being used
  52    * @var string
  53    */
  54   protected $_seperator;
  55
  56   /**
  57    * Total number of lines in file
  58    * @var int
  59    */
  60   protected $_lineCount;
  61
  62   /**
  63    * Whether the file has a column header or not
  64    *
  65    * @var boolean
  66    */
  67   protected $_haveColumnHeader;
  68
  69   /**
  70    * @param string $fileName
  71    * @param string $seperator
  72    * @param $mapper
  73    * @param bool $skipColumnHeader
  74    * @param int $mode
  75    * @param int $contactType
  76    * @param int $onDuplicate
  77    * @param int $statusID
  78    * @param int $totalRowCount
  79    *
  80    * @return mixed
  81    * @throws Exception
  82    */
  83   public function run(
  84     $fileName,
  85     $seperator = ',',
  86     &$mapper,
  87     $skipColumnHeader = FALSE,
  88     $mode = self::MODE_PREVIEW,
  89     $contactType = self::CONTACT_INDIVIDUAL,
  90     $onDuplicate = self::DUPLICATE_SKIP,
  91     $statusID = NULL,
  92     $totalRowCount = NULL
  93   ) {
  94     if (!is_array($fileName)) {
  95       CRM_Core_Error::fatal();
  96     }
  97     $fileName = $fileName['name'];
  98
  99     switch ($contactType) {
 100       case self::CONTACT_INDIVIDUAL:
 101         $this->_contactType = 'Individual';
 102         break;
 103
 104       case self::CONTACT_HOUSEHOLD:
 105         $this->_contactType = 'Household';
 106         break;
 107
 108       case self::CONTACT_ORGANIZATION:
 109         $this->_contactType = 'Organization';
 110     }
 111
 112     $this->init();
 113
 114     $this->_haveColumnHeader = $skipColumnHeader;
 115
 116     $this->_seperator = $seperator;
 117
 118     $fd = fopen($fileName, "r");
 119     if (!$fd) {
 120       return FALSE;
 121     }
 122
 123     $this->_lineCount = $this->_warningCount = 0;
 124     $this->_invalidRowCount = $this->_validCount = 0;
 125     $this->_totalCount = $this->_conflictCount = 0;
 126
 127     $this->_errors = [];
 128     $this->_warnings = [];
 129     $this->_conflicts = [];
 130
 131     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 132
 133     if ($mode == self::MODE_MAPFIELD) {
 134       $this->_rows = [];
 135     }
 136     else {
 137       $this->_activeFieldCount = count($this->_activeFields);
 138     }
 139     if ($statusID) {
 140       $this->progressImport($statusID);
 141       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 142     }
 143
 144     while (!feof($fd)) {
 145       $this->_lineCount++;
 146
 147       $values = fgetcsv($fd, 8192, $seperator);
 148       if (!$values) {
 149         continue;
 150       }
 151
 152       self::encloseScrub($values);
 153
 154       // skip column header if we're not in mapfield mode
 155       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 156         $skipColumnHeader = FALSE;
 157         continue;
 158       }
 159
 160       /* trim whitespace around the values */
 161       $empty = TRUE;
 162       foreach ($values as $k => $v) {
 163         $values[$k] = trim($v, " \t\r\n");
 164       }
 165       if (CRM_Utils_System::isNull($values)) {
 166         continue;
 167       }
 168
 169       $this->_totalCount++;
 170
 171       if ($mode == self::MODE_MAPFIELD) {
 172         $returnCode = $this->mapField($values);
 173       }
 174       elseif ($mode == self::MODE_PREVIEW) {
 175         $returnCode = $this->preview($values);
 176       }
 177       elseif ($mode == self::MODE_SUMMARY) {
 178         $returnCode = $this->summary($values);
 179       }
 180       elseif ($mode == self::MODE_IMPORT) {
 181         $returnCode = $this->import($onDuplicate, $values);
 182         if ($statusID && (($this->_lineCount % 50) == 0)) {
 183           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 184         }
 185       }
 186       else {
 187         $returnCode = self::ERROR;
 188       }
 189
 190       // note that a line could be valid but still produce a warning
 191       if ($returnCode & self::VALID) {
 192         $this->_validCount++;
 193         if ($mode == self::MODE_MAPFIELD) {
 194           $this->_rows[] = $values;
 195           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 196         }
 197       }
 198
 199       if ($returnCode & self::WARNING) {
 200         $this->_warningCount++;
 201         if ($this->_warningCount < $this->_maxWarningCount) {
 202           $this->_warningCount[] = $line;
 203         }
 204       }
 205
 206       if ($returnCode & self::ERROR) {
 207         $this->_invalidRowCount++;
 208         $recordNumber = $this->_lineCount;
 209         array_unshift($values, $recordNumber);
 210         $this->_errors[] = $values;
 211       }
 212
 213       if ($returnCode & self::CONFLICT) {
 214         $this->_conflictCount++;
 215         $recordNumber = $this->_lineCount;
 216         array_unshift($values, $recordNumber);
 217         $this->_conflicts[] = $values;
 218       }
 219
 220       if ($returnCode & self::DUPLICATE) {
 221         if ($returnCode & self::MULTIPLE_DUPE) {
 222           /* TODO: multi-dupes should be counted apart from singles
 223            * on non-skip action */
 224         }
 225         $this->_duplicateCount++;
 226         $recordNumber = $this->_lineCount;
 227         array_unshift($values, $recordNumber);
 228         $this->_duplicates[] = $values;
 229         if ($onDuplicate != self::DUPLICATE_SKIP) {
 230           $this->_validCount++;
 231         }
 232       }
 233
 234       // we give the derived class a way of aborting the process
 235       // note that the return code could be multiple code or'ed together
 236       if ($returnCode & self::STOP) {
 237         break;
 238       }
 239
 240       // if we are done processing the maxNumber of lines, break
 241       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 242         break;
 243       }
 244     }
 245
 246     fclose($fd);
 247
 248     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 249       $customHeaders = $mapper;
 250
 251       $customfields = CRM_Core_BAO_CustomField::getFields('Membership');
 252       foreach ($customHeaders as $key => $value) {
 253         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 254           $customHeaders[$key] = $customfields[$id][0];
 255         }
 256       }
 257       if ($this->_invalidRowCount) {
 258         // removed view url for invlaid contacts
 259         $headers = array_merge([
 260           ts('Line Number'),
 261           ts('Reason'),
 262         ], $customHeaders);
 263         $this->_errorFileName = self::errorFileName(self::ERROR);
 264
 265         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 266       }
 267       if ($this->_conflictCount) {
 268         $headers = array_merge([
 269           ts('Line Number'),
 270           ts('Reason'),
 271         ], $customHeaders);
 272         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 273         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 274       }
 275       if ($this->_duplicateCount) {
 276         $headers = array_merge([
 277           ts('Line Number'),
 278           ts('View Membership URL'),
 279         ], $customHeaders);
 280
 281         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 282         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 283       }
 284     }
 285     return $this->fini();
 286   }
 287
 288   /**
 289    * Given a list of the importable field keys that the user has selected
 290    * set the active fields array to this list
 291    *
 292    * @param array $fieldKeys mapped array of values
 293    *
 294    * @return void
 295    */
 296   public function setActiveFields($fieldKeys) {
 297     $this->_activeFieldCount = count($fieldKeys);
 298     foreach ($fieldKeys as $key) {
 299       if (empty($this->_fields[$key])) {
 300         $this->_activeFields[] = new CRM_Member_Import_Field('', ts('- do not import -'));
 301       }
 302       else {
 303         $this->_activeFields[] = clone($this->_fields[$key]);
 304       }
 305     }
 306   }
 307
 308   /**
 309    * Format the field values for input to the api.
 310    *
 311    * @return array
 312    *   (reference ) associative array of name/value pairs
 313    */
 314   public function &getActiveFieldParams() {
 315     $params = [];
 316     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 317       if (isset($this->_activeFields[$i]->_value)
 318         && !isset($params[$this->_activeFields[$i]->_name])
 319         && !isset($this->_activeFields[$i]->_related)
 320       ) {
 321
 322         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 323       }
 324     }
 325     return $params;
 326   }
 327
 328   /**
 329    * @param string $name
 330    * @param $title
 331    * @param int $type
 332    * @param string $headerPattern
 333    * @param string $dataPattern
 334    */
 335   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 336     if (empty($name)) {
 337       $this->_fields['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 338     }
 339     else {
 340
 341       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 342       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 343       if (!array_key_exists($name, $tempField)) {
 344         $this->_fields[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 345       }
 346       else {
 347         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 348           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 349         );
 350       }
 351     }
 352   }
 353
 354   /**
 355    * Store parser values.
 356    *
 357    * @param CRM_Core_Session $store
 358    *
 359    * @param int $mode
 360    *
 361    * @return void
 362    */
 363   public function set($store, $mode = self::MODE_SUMMARY) {
 364     $store->set('fileSize', $this->_fileSize);
 365     $store->set('lineCount', $this->_lineCount);
 366     $store->set('seperator', $this->_seperator);
 367     $store->set('fields', $this->getSelectValues());
 368     $store->set('fieldTypes', $this->getSelectTypes());
 369
 370     $store->set('headerPatterns', $this->getHeaderPatterns());
 371     $store->set('dataPatterns', $this->getDataPatterns());
 372     $store->set('columnCount', $this->_activeFieldCount);
 373
 374     $store->set('totalRowCount', $this->_totalCount);
 375     $store->set('validRowCount', $this->_validCount);
 376     $store->set('invalidRowCount', $this->_invalidRowCount);
 377     $store->set('conflictRowCount', $this->_conflictCount);
 378
 379     switch ($this->_contactType) {
 380       case 'Individual':
 381         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 382         break;
 383
 384       case 'Household':
 385         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 386         break;
 387
 388       case 'Organization':
 389         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 390     }
 391
 392     if ($this->_invalidRowCount) {
 393       $store->set('errorsFileName', $this->_errorFileName);
 394     }
 395     if ($this->_conflictCount) {
 396       $store->set('conflictsFileName', $this->_conflictFileName);
 397     }
 398     if (isset($this->_rows) && !empty($this->_rows)) {
 399       $store->set('dataValues', $this->_rows);
 400     }
 401
 402     if ($mode == self::MODE_IMPORT) {
 403       $store->set('duplicateRowCount', $this->_duplicateCount);
 404       if ($this->_duplicateCount) {
 405         $store->set('duplicatesFileName', $this->_duplicateFileName);
 406       }
 407     }
 408   }
 409
 410   /**
 411    * Export data to a CSV file.
 412    *
 413    * @param string $fileName
 414    * @param array $header
 415    * @param array $data
 416    *
 417    * @return void
 418    */
 419   public static function exportCSV($fileName, $header, $data) {
 420     $output = [];
 421     $fd = fopen($fileName, 'w');
 422
 423     foreach ($header as $key => $value) {
 424       $header[$key] = "\"$value\"";
 425     }
 426     $config = CRM_Core_Config::singleton();
 427     $output[] = implode($config->fieldSeparator, $header);
 428
 429     foreach ($data as $datum) {
 430       foreach ($datum as $key => $value) {
 431         if (is_array($value)) {
 432           foreach ($value[0] as $k1 => $v1) {
 433             if ($k1 == 'location_type_id') {
 434               continue;
 435             }
 436             $datum[$k1] = $v1;
 437           }
 438         }
 439         else {
 440           $datum[$key] = "\"$value\"";
 441         }
 442       }
 443       $output[] = implode($config->fieldSeparator, $datum);
 444     }
 445     fwrite($fd, implode("\n", $output));
 446     fclose($fd);
 447   }
 448
 449 }