CRM/Custom/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2015                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2015
  32  * $Id$
  33  *
  34  */
  35 abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
  36
  37   protected $_fileName;
  38
  39   /**#@+
  40    * @var integer
  41    */
  42
  43   /**
  44    * Imported file size
  45    */
  46   protected $_fileSize;
  47
  48   /**
  49    * Separator being used
  50    */
  51   protected $_separator;
  52
  53   /**
  54    * Total number of lines in file
  55    */
  56   protected $_lineCount;
  57
  58   /**
  59    * Whether the file has a column header or not
  60    *
  61    * @var boolean
  62    */
  63   protected $_haveColumnHeader;
  64
  65   /**
  66    * @param string $fileName
  67    * @param string $separator
  68    * @param int $mapper
  69    * @param bool $skipColumnHeader
  70    * @param int|string $mode
  71    * @param int|string $contactType
  72    * @param int $onDuplicate
  73    *
  74    * @return mixed
  75    * @throws Exception
  76    */
  77   public function run(
  78     $fileName,
  79     $separator = ',',
  80     &$mapper,
  81     $skipColumnHeader = FALSE,
  82     $mode = self::MODE_PREVIEW,
  83     $contactType = self::CONTACT_INDIVIDUAL,
  84     $onDuplicate = self::DUPLICATE_SKIP
  85   ) {
  86     if (!is_array($fileName)) {
  87       CRM_Core_Error::fatal();
  88     }
  89     $fileName = $fileName['name'];
  90
  91     switch ($contactType) {
  92       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
  93         $this->_contactType = 'Individual';
  94         break;
  95
  96       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
  97         $this->_contactType = 'Household';
  98         break;
  99
 100       case CRM_Import_Parser::CONTACT_ORGANIZATION:
 101         $this->_contactType = 'Organization';
 102     }
 103     $this->init();
 104
 105     $this->_haveColumnHeader = $skipColumnHeader;
 106
 107     $this->_separator = $separator;
 108
 109     $fd = fopen($fileName, "r");
 110     if (!$fd) {
 111       return FALSE;
 112     }
 113
 114     $this->_lineCount = $this->_warningCount = 0;
 115     $this->_invalidRowCount = $this->_validCount = 0;
 116     $this->_totalCount = $this->_conflictCount = 0;
 117
 118     $this->_errors = array();
 119     $this->_warnings = array();
 120     $this->_conflicts = array();
 121
 122     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 123
 124     if ($mode == self::MODE_MAPFIELD) {
 125       $this->_rows = array();
 126     }
 127     else {
 128       $this->_activeFieldCount = count($this->_activeFields);
 129     }
 130
 131     while (!feof($fd)) {
 132       $this->_lineCount++;
 133
 134       $values = fgetcsv($fd, 8192, $separator);
 135       if (!$values) {
 136         continue;
 137       }
 138
 139       self::encloseScrub($values);
 140
 141       // skip column header if we're not in mapfield mode
 142       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 143         $skipColumnHeader = FALSE;
 144         continue;
 145       }
 146
 147       /* trim whitespace around the values */
 148
 149       $empty = TRUE;
 150       foreach ($values as $k => $v) {
 151         $values[$k] = trim($v, " \t\r\n");
 152       }
 153
 154       if (CRM_Utils_System::isNull($values)) {
 155         continue;
 156       }
 157
 158       $this->_totalCount++;
 159
 160       if ($mode == self::MODE_MAPFIELD) {
 161         $returnCode = $this->mapField($values);
 162       }
 163       elseif ($mode == self::MODE_PREVIEW) {
 164         $returnCode = $this->preview($values);
 165       }
 166       elseif ($mode == self::MODE_SUMMARY) {
 167         $returnCode = $this->summary($values);
 168       }
 169       elseif ($mode == self::MODE_IMPORT) {
 170         $returnCode = $this->import($onDuplicate, $values);
 171       }
 172       else {
 173         $returnCode = self::ERROR;
 174       }
 175
 176       // note that a line could be valid but still produce a warning
 177       if ($returnCode & self::VALID) {
 178         $this->_validCount++;
 179         if ($mode == self::MODE_MAPFIELD) {
 180           $this->_rows[] = $values;
 181           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 182         }
 183       }
 184
 185       if ($returnCode & self::WARNING) {
 186         $this->_warningCount++;
 187         if ($this->_warningCount < $this->_maxWarningCount) {
 188           $this->_warningCount[] = $line;
 189         }
 190       }
 191
 192       if ($returnCode & self::ERROR) {
 193         $this->_invalidRowCount++;
 194         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 195           $recordNumber = $this->_lineCount;
 196           if ($this->_haveColumnHeader) {
 197             $recordNumber--;
 198           }
 199           array_unshift($values, $recordNumber);
 200           $this->_errors[] = $values;
 201         }
 202       }
 203
 204       if ($returnCode & self::CONFLICT) {
 205         $this->_conflictCount++;
 206         $recordNumber = $this->_lineCount;
 207         if ($this->_haveColumnHeader) {
 208           $recordNumber--;
 209         }
 210         array_unshift($values, $recordNumber);
 211         $this->_conflicts[] = $values;
 212       }
 213
 214       if ($returnCode & self::DUPLICATE) {
 215         if ($returnCode & self::MULTIPLE_DUPE) {
 216           /* TODO: multi-dupes should be counted apart from singles
 217            * on non-skip action */
 218         }
 219         $this->_duplicateCount++;
 220         $recordNumber = $this->_lineCount;
 221         if ($this->_haveColumnHeader) {
 222           $recordNumber--;
 223         }
 224         array_unshift($values, $recordNumber);
 225         $this->_duplicates[] = $values;
 226         if ($onDuplicate != self::DUPLICATE_SKIP) {
 227           $this->_validCount++;
 228         }
 229       }
 230
 231       // we give the derived class a way of aborting the process
 232       // note that the return code could be multiple code or'ed together
 233       if ($returnCode & self::STOP) {
 234         break;
 235       }
 236
 237       // if we are done processing the maxNumber of lines, break
 238       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 239         break;
 240       }
 241     }
 242
 243     fclose($fd);
 244
 245     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 246       $customHeaders = $mapper;
 247
 248       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 249       foreach ($customHeaders as $key => $value) {
 250         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 251           $customHeaders[$key] = $customfields[$id][0];
 252         }
 253       }
 254       if ($this->_invalidRowCount) {
 255         // removed view url for invlaid contacts
 256         $headers = array_merge(array(
 257             ts('Line Number'),
 258             ts('Reason'),
 259           ),
 260           $customHeaders
 261         );
 262         $this->_errorFileName = self::errorFileName(self::ERROR);
 263         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 264       }
 265       if ($this->_conflictCount) {
 266         $headers = array_merge(array(
 267             ts('Line Number'),
 268             ts('Reason'),
 269           ),
 270           $customHeaders
 271         );
 272         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 273         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 274       }
 275       if ($this->_duplicateCount) {
 276         $headers = array_merge(array(
 277             ts('Line Number'),
 278             ts('View Activity History URL'),
 279           ),
 280           $customHeaders
 281         );
 282
 283         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 284         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 285       }
 286     }
 287     return $this->fini();
 288   }
 289
 290   /**
 291    * Given a list of the importable field keys that the user has selected
 292    * set the active fields array to this list
 293    *
 294    * @param array $fieldKeys mapped array of values
 295    *
 296    * @return void
 297    */
 298   public function setActiveFields($fieldKeys) {
 299     $this->_activeFieldCount = count($fieldKeys);
 300     foreach ($fieldKeys as $key) {
 301       if (empty($this->_fields[$key])) {
 302         $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
 303       }
 304       else {
 305         $this->_activeFields[] = clone($this->_fields[$key]);
 306       }
 307     }
 308   }
 309
 310   /**
 311    * Format the field values for input to the api.
 312    *
 313    * @return array
 314    *   (reference ) associative array of name/value pairs
 315    */
 316   public function &getActiveFieldParams() {
 317     $params = array();
 318     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 319       if (isset($this->_activeFields[$i]->_value)
 320         && !isset($params[$this->_activeFields[$i]->_name])
 321         && !isset($this->_activeFields[$i]->_related)
 322       ) {
 323         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 324       }
 325     }
 326     return $params;
 327   }
 328
 329   /**
 330    * Store parser values.
 331    *
 332    * @param CRM_Core_Session $store
 333    *
 334    * @param int $mode
 335    *
 336    * @return void
 337    */
 338   public function set($store, $mode = self::MODE_SUMMARY) {
 339     $store->set('fileSize', $this->_fileSize);
 340     $store->set('lineCount', $this->_lineCount);
 341     $store->set('seperator', $this->_separator);
 342     $store->set('fields', $this->getSelectValues());
 343     $store->set('fieldTypes', $this->getSelectTypes());
 344
 345     $store->set('headerPatterns', $this->getHeaderPatterns());
 346     $store->set('dataPatterns', $this->getDataPatterns());
 347     $store->set('columnCount', $this->_activeFieldCount);
 348     $store->set('_entity', $this->_entity);
 349     $store->set('totalRowCount', $this->_totalCount);
 350     $store->set('validRowCount', $this->_validCount);
 351     $store->set('invalidRowCount', $this->_invalidRowCount);
 352     $store->set('conflictRowCount', $this->_conflictCount);
 353
 354     switch ($this->_contactType) {
 355       case 'Individual':
 356         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 357         break;
 358
 359       case 'Household':
 360         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 361         break;
 362
 363       case 'Organization':
 364         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 365     }
 366
 367     if ($this->_invalidRowCount) {
 368       $store->set('errorsFileName', $this->_errorFileName);
 369     }
 370     if ($this->_conflictCount) {
 371       $store->set('conflictsFileName', $this->_conflictFileName);
 372     }
 373     if (isset($this->_rows) && !empty($this->_rows)) {
 374       $store->set('dataValues', $this->_rows);
 375     }
 376
 377     if ($mode == self::MODE_IMPORT) {
 378       $store->set('duplicateRowCount', $this->_duplicateCount);
 379       if ($this->_duplicateCount) {
 380         $store->set('duplicatesFileName', $this->_duplicateFileName);
 381       }
 382     }
 383   }
 384
 385 }