CRM/Member/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2018                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2018
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Member_Import_Parser extends CRM_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**#@+
  40    * @var integer
  41    */
  42
  43   /**
  44    * Imported file size
  45    */
  46   protected $_fileSize;
  47
  48   /**
  49    * Seperator being used
  50    */
  51   protected $_seperator;
  52
  53   /**
  54    * Total number of lines in file
  55    */
  56   protected $_lineCount;
  57
  58   /**
  59    * Whether the file has a column header or not
  60    *
  61    * @var boolean
  62    */
  63   protected $_haveColumnHeader;
  64
  65   /**
  66    * @param string $fileName
  67    * @param string $seperator
  68    * @param $mapper
  69    * @param bool $skipColumnHeader
  70    * @param int $mode
  71    * @param int $contactType
  72    * @param int $onDuplicate
  73    *
  74    * @return mixed
  75    * @throws Exception
  76    */
  77   public function run(
  78     $fileName,
  79     $seperator = ',',
  80     &$mapper,
  81     $skipColumnHeader = FALSE,
  82     $mode = self::MODE_PREVIEW,
  83     $contactType = self::CONTACT_INDIVIDUAL,
  84     $onDuplicate = self::DUPLICATE_SKIP,
  85     $statusID = NULL,
  86     $totalRowCount = NULL
  87   ) {
  88     if (!is_array($fileName)) {
  89       CRM_Core_Error::fatal();
  90     }
  91     $fileName = $fileName['name'];
  92
  93     switch ($contactType) {
  94       case self::CONTACT_INDIVIDUAL:
  95         $this->_contactType = 'Individual';
  96         break;
  97
  98       case self::CONTACT_HOUSEHOLD:
  99         $this->_contactType = 'Household';
 100         break;
 101
 102       case self::CONTACT_ORGANIZATION:
 103         $this->_contactType = 'Organization';
 104     }
 105
 106     $this->init();
 107
 108     $this->_haveColumnHeader = $skipColumnHeader;
 109
 110     $this->_seperator = $seperator;
 111
 112     $fd = fopen($fileName, "r");
 113     if (!$fd) {
 114       return FALSE;
 115     }
 116
 117     $this->_lineCount = $this->_warningCount = 0;
 118     $this->_invalidRowCount = $this->_validCount = 0;
 119     $this->_totalCount = $this->_conflictCount = 0;
 120
 121     $this->_errors = array();
 122     $this->_warnings = array();
 123     $this->_conflicts = array();
 124
 125     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 126
 127     if ($mode == self::MODE_MAPFIELD) {
 128       $this->_rows = array();
 129     }
 130     else {
 131       $this->_activeFieldCount = count($this->_activeFields);
 132     }
 133     if ($statusID) {
 134       $this->progressImport($statusID);
 135       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 136     }
 137
 138     while (!feof($fd)) {
 139       $this->_lineCount++;
 140
 141       $values = fgetcsv($fd, 8192, $seperator);
 142       if (!$values) {
 143         continue;
 144       }
 145
 146       self::encloseScrub($values);
 147
 148       // skip column header if we're not in mapfield mode
 149       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 150         $skipColumnHeader = FALSE;
 151         continue;
 152       }
 153
 154       /* trim whitespace around the values */
 155       $empty = TRUE;
 156       foreach ($values as $k => $v) {
 157         $values[$k] = trim($v, " \t\r\n");
 158       }
 159       if (CRM_Utils_System::isNull($values)) {
 160         continue;
 161       }
 162
 163       $this->_totalCount++;
 164
 165       if ($mode == self::MODE_MAPFIELD) {
 166         $returnCode = $this->mapField($values);
 167       }
 168       elseif ($mode == self::MODE_PREVIEW) {
 169         $returnCode = $this->preview($values);
 170       }
 171       elseif ($mode == self::MODE_SUMMARY) {
 172         $returnCode = $this->summary($values);
 173       }
 174       elseif ($mode == self::MODE_IMPORT) {
 175         $returnCode = $this->import($onDuplicate, $values);
 176         if ($statusID && (($this->_lineCount % 50) == 0)) {
 177           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 178         }
 179       }
 180       else {
 181         $returnCode = self::ERROR;
 182       }
 183
 184       // note that a line could be valid but still produce a warning
 185       if ($returnCode & self::VALID) {
 186         $this->_validCount++;
 187         if ($mode == self::MODE_MAPFIELD) {
 188           $this->_rows[] = $values;
 189           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 190         }
 191       }
 192
 193       if ($returnCode & self::WARNING) {
 194         $this->_warningCount++;
 195         if ($this->_warningCount < $this->_maxWarningCount) {
 196           $this->_warningCount[] = $line;
 197         }
 198       }
 199
 200       if ($returnCode & self::ERROR) {
 201         $this->_invalidRowCount++;
 202         $recordNumber = $this->_lineCount;
 203         array_unshift($values, $recordNumber);
 204         $this->_errors[] = $values;
 205       }
 206
 207       if ($returnCode & self::CONFLICT) {
 208         $this->_conflictCount++;
 209         $recordNumber = $this->_lineCount;
 210         array_unshift($values, $recordNumber);
 211         $this->_conflicts[] = $values;
 212       }
 213
 214       if ($returnCode & self::DUPLICATE) {
 215         if ($returnCode & self::MULTIPLE_DUPE) {
 216           /* TODO: multi-dupes should be counted apart from singles
 217            * on non-skip action */
 218         }
 219         $this->_duplicateCount++;
 220         $recordNumber = $this->_lineCount;
 221         array_unshift($values, $recordNumber);
 222         $this->_duplicates[] = $values;
 223         if ($onDuplicate != self::DUPLICATE_SKIP) {
 224           $this->_validCount++;
 225         }
 226       }
 227
 228       // we give the derived class a way of aborting the process
 229       // note that the return code could be multiple code or'ed together
 230       if ($returnCode & self::STOP) {
 231         break;
 232       }
 233
 234       // if we are done processing the maxNumber of lines, break
 235       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 236         break;
 237       }
 238     }
 239
 240     fclose($fd);
 241
 242     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 243       $customHeaders = $mapper;
 244
 245       $customfields = CRM_Core_BAO_CustomField::getFields('Membership');
 246       foreach ($customHeaders as $key => $value) {
 247         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 248           $customHeaders[$key] = $customfields[$id][0];
 249         }
 250       }
 251       if ($this->_invalidRowCount) {
 252         // removed view url for invlaid contacts
 253         $headers = array_merge(array(
 254           ts('Line Number'),
 255           ts('Reason'),
 256         ), $customHeaders);
 257         $this->_errorFileName = self::errorFileName(self::ERROR);
 258
 259         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 260       }
 261       if ($this->_conflictCount) {
 262         $headers = array_merge(array(
 263           ts('Line Number'),
 264           ts('Reason'),
 265         ), $customHeaders);
 266         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 267         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 268       }
 269       if ($this->_duplicateCount) {
 270         $headers = array_merge(array(
 271           ts('Line Number'),
 272           ts('View Membership URL'),
 273         ), $customHeaders);
 274
 275         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 276         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 277       }
 278     }
 279     return $this->fini();
 280   }
 281
 282   /**
 283    * Given a list of the importable field keys that the user has selected
 284    * set the active fields array to this list
 285    *
 286    * @param array $fieldKeys mapped array of values
 287    *
 288    * @return void
 289    */
 290   public function setActiveFields($fieldKeys) {
 291     $this->_activeFieldCount = count($fieldKeys);
 292     foreach ($fieldKeys as $key) {
 293       if (empty($this->_fields[$key])) {
 294         $this->_activeFields[] = new CRM_Member_Import_Field('', ts('- do not import -'));
 295       }
 296       else {
 297         $this->_activeFields[] = clone($this->_fields[$key]);
 298       }
 299     }
 300   }
 301
 302   /**
 303    * Format the field values for input to the api.
 304    *
 305    * @return array
 306    *   (reference ) associative array of name/value pairs
 307    */
 308   public function &getActiveFieldParams() {
 309     $params = array();
 310     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 311       if (isset($this->_activeFields[$i]->_value)
 312         && !isset($params[$this->_activeFields[$i]->_name])
 313         && !isset($this->_activeFields[$i]->_related)
 314       ) {
 315
 316         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 317       }
 318     }
 319     return $params;
 320   }
 321
 322   /**
 323    * @param string $name
 324    * @param $title
 325    * @param int $type
 326    * @param string $headerPattern
 327    * @param string $dataPattern
 328    */
 329   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 330     if (empty($name)) {
 331       $this->_fields['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 332     }
 333     else {
 334
 335       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 336       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 337       if (!array_key_exists($name, $tempField)) {
 338         $this->_fields[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 339       }
 340       else {
 341         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 342           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 343         );
 344       }
 345     }
 346   }
 347
 348   /**
 349    * Store parser values.
 350    *
 351    * @param CRM_Core_Session $store
 352    *
 353    * @param int $mode
 354    *
 355    * @return void
 356    */
 357   public function set($store, $mode = self::MODE_SUMMARY) {
 358     $store->set('fileSize', $this->_fileSize);
 359     $store->set('lineCount', $this->_lineCount);
 360     $store->set('seperator', $this->_seperator);
 361     $store->set('fields', $this->getSelectValues());
 362     $store->set('fieldTypes', $this->getSelectTypes());
 363
 364     $store->set('headerPatterns', $this->getHeaderPatterns());
 365     $store->set('dataPatterns', $this->getDataPatterns());
 366     $store->set('columnCount', $this->_activeFieldCount);
 367
 368     $store->set('totalRowCount', $this->_totalCount);
 369     $store->set('validRowCount', $this->_validCount);
 370     $store->set('invalidRowCount', $this->_invalidRowCount);
 371     $store->set('conflictRowCount', $this->_conflictCount);
 372
 373     switch ($this->_contactType) {
 374       case 'Individual':
 375         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 376         break;
 377
 378       case 'Household':
 379         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 380         break;
 381
 382       case 'Organization':
 383         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 384     }
 385
 386     if ($this->_invalidRowCount) {
 387       $store->set('errorsFileName', $this->_errorFileName);
 388     }
 389     if ($this->_conflictCount) {
 390       $store->set('conflictsFileName', $this->_conflictFileName);
 391     }
 392     if (isset($this->_rows) && !empty($this->_rows)) {
 393       $store->set('dataValues', $this->_rows);
 394     }
 395
 396     if ($mode == self::MODE_IMPORT) {
 397       $store->set('duplicateRowCount', $this->_duplicateCount);
 398       if ($this->_duplicateCount) {
 399         $store->set('duplicatesFileName', $this->_duplicateFileName);
 400       }
 401     }
 402   }
 403
 404   /**
 405    * Export data to a CSV file.
 406    *
 407    * @param string $fileName
 408    * @param array $header
 409    * @param array $data
 410    *
 411    * @return void
 412    */
 413   public static function exportCSV($fileName, $header, $data) {
 414     $output = array();
 415     $fd = fopen($fileName, 'w');
 416
 417     foreach ($header as $key => $value) {
 418       $header[$key] = "\"$value\"";
 419     }
 420     $config = CRM_Core_Config::singleton();
 421     $output[] = implode($config->fieldSeparator, $header);
 422
 423     foreach ($data as $datum) {
 424       foreach ($datum as $key => $value) {
 425         if (is_array($value)) {
 426           foreach ($value[0] as $k1 => $v1) {
 427             if ($k1 == 'location_type_id') {
 428               continue;
 429             }
 430             $datum[$k1] = $v1;
 431           }
 432         }
 433         else {
 434           $datum[$key] = "\"$value\"";
 435         }
 436       }
 437       $output[] = implode($config->fieldSeparator, $datum);
 438     }
 439     fwrite($fd, implode("\n", $output));
 440     fclose($fd);
 441   }
 442
 443 }