CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2018                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2018
  32  */
  33 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  34
  35   /**
  36    * Contribution-specific result codes
  37    * @see CRM_Import_Parser result code constants
  38    */
  39   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  40
  41   protected $_fileName;
  42
  43   /**
  44    * Imported file size
  45    */
  46   protected $_fileSize;
  47
  48   /**
  49    * Seperator being used
  50    */
  51   protected $_seperator;
  52
  53   /**
  54    * Total number of lines in file
  55    */
  56   protected $_lineCount;
  57
  58   /**
  59    * Running total number of valid soft credit rows
  60    */
  61   protected $_validSoftCreditRowCount;
  62
  63   /**
  64    * Running total number of invalid soft credit rows
  65    */
  66   protected $_invalidSoftCreditRowCount;
  67
  68   /**
  69    * Running total number of valid pledge payment rows
  70    */
  71   protected $_validPledgePaymentRowCount;
  72
  73   /**
  74    * Running total number of invalid pledge payment rows
  75    */
  76   protected $_invalidPledgePaymentRowCount;
  77
  78   /**
  79    * Array of pledge payment error lines, bounded by MAX_ERROR
  80    */
  81   protected $_pledgePaymentErrors;
  82
  83   /**
  84    * Array of pledge payment error lines, bounded by MAX_ERROR
  85    */
  86   protected $_softCreditErrors;
  87
  88   /**
  89    * Filename of pledge payment error data
  90    *
  91    * @var string
  92    */
  93   protected $_pledgePaymentErrorsFileName;
  94
  95   /**
  96    * Filename of soft credit error data
  97    *
  98    * @var string
  99    */
 100   protected $_softCreditErrorsFileName;
 101
 102   /**
 103    * Whether the file has a column header or not
 104    *
 105    * @var boolean
 106    */
 107   protected $_haveColumnHeader;
 108
 109   /**
 110    * @param string $fileName
 111    * @param string $seperator
 112    * @param $mapper
 113    * @param bool $skipColumnHeader
 114    * @param int $mode
 115    * @param int $contactType
 116    * @param int $onDuplicate
 117    *
 118    * @return mixed
 119    * @throws Exception
 120    */
 121   public function run(
 122     $fileName,
 123     $seperator = ',',
 124     &$mapper,
 125     $skipColumnHeader = FALSE,
 126     $mode = self::MODE_PREVIEW,
 127     $contactType = self::CONTACT_INDIVIDUAL,
 128     $onDuplicate = self::DUPLICATE_SKIP,
 129     $statusID = NULL,
 130     $totalRowCount = NULL
 131   ) {
 132     if (!is_array($fileName)) {
 133       CRM_Core_Error::fatal();
 134     }
 135     $fileName = $fileName['name'];
 136
 137     switch ($contactType) {
 138       case self::CONTACT_INDIVIDUAL:
 139         $this->_contactType = 'Individual';
 140         break;
 141
 142       case self::CONTACT_HOUSEHOLD:
 143         $this->_contactType = 'Household';
 144         break;
 145
 146       case self::CONTACT_ORGANIZATION:
 147         $this->_contactType = 'Organization';
 148     }
 149
 150     $this->init();
 151
 152     $this->_haveColumnHeader = $skipColumnHeader;
 153
 154     $this->_seperator = $seperator;
 155
 156     $fd = fopen($fileName, "r");
 157     if (!$fd) {
 158       return FALSE;
 159     }
 160
 161     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 162     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 163     $this->_totalCount = $this->_conflictCount = 0;
 164
 165     $this->_errors = array();
 166     $this->_warnings = array();
 167     $this->_conflicts = array();
 168     $this->_pledgePaymentErrors = array();
 169     $this->_softCreditErrors = array();
 170     if ($statusID) {
 171       $this->progressImport($statusID);
 172       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 173     }
 174
 175     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 176
 177     if ($mode == self::MODE_MAPFIELD) {
 178       $this->_rows = array();
 179     }
 180     else {
 181       $this->_activeFieldCount = count($this->_activeFields);
 182     }
 183
 184     while (!feof($fd)) {
 185       $this->_lineCount++;
 186
 187       $values = fgetcsv($fd, 8192, $seperator);
 188       if (!$values) {
 189         continue;
 190       }
 191
 192       self::encloseScrub($values);
 193
 194       // skip column header if we're not in mapfield mode
 195       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 196         $skipColumnHeader = FALSE;
 197         continue;
 198       }
 199
 200       /* trim whitespace around the values */
 201
 202       $empty = TRUE;
 203       foreach ($values as $k => $v) {
 204         $values[$k] = trim($v, " \t\r\n");
 205       }
 206
 207       if (CRM_Utils_System::isNull($values)) {
 208         continue;
 209       }
 210
 211       $this->_totalCount++;
 212
 213       if ($mode == self::MODE_MAPFIELD) {
 214         $returnCode = $this->mapField($values);
 215       }
 216       elseif ($mode == self::MODE_PREVIEW) {
 217         $returnCode = $this->preview($values);
 218       }
 219       elseif ($mode == self::MODE_SUMMARY) {
 220         $returnCode = $this->summary($values);
 221       }
 222       elseif ($mode == self::MODE_IMPORT) {
 223         $returnCode = $this->import($onDuplicate, $values);
 224         if ($statusID && (($this->_lineCount % 50) == 0)) {
 225           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 226         }
 227       }
 228       else {
 229         $returnCode = self::ERROR;
 230       }
 231
 232       // note that a line could be valid but still produce a warning
 233       if ($returnCode == self::VALID) {
 234         $this->_validCount++;
 235         if ($mode == self::MODE_MAPFIELD) {
 236           $this->_rows[] = $values;
 237           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 238         }
 239       }
 240
 241       if ($returnCode == self::SOFT_CREDIT) {
 242         $this->_validSoftCreditRowCount++;
 243         $this->_validCount++;
 244         if ($mode == self::MODE_MAPFIELD) {
 245           $this->_rows[] = $values;
 246           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 247         }
 248       }
 249
 250       if ($returnCode == self::PLEDGE_PAYMENT) {
 251         $this->_validPledgePaymentRowCount++;
 252         $this->_validCount++;
 253         if ($mode == self::MODE_MAPFIELD) {
 254           $this->_rows[] = $values;
 255           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 256         }
 257       }
 258
 259       if ($returnCode == self::WARNING) {
 260         $this->_warningCount++;
 261         if ($this->_warningCount < $this->_maxWarningCount) {
 262           $this->_warningCount[] = $line;
 263         }
 264       }
 265
 266       if ($returnCode == self::ERROR) {
 267         $this->_invalidRowCount++;
 268         $recordNumber = $this->_lineCount;
 269         if ($this->_haveColumnHeader) {
 270           $recordNumber--;
 271         }
 272         array_unshift($values, $recordNumber);
 273         $this->_errors[] = $values;
 274       }
 275
 276       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 277         $this->_invalidPledgePaymentRowCount++;
 278         $recordNumber = $this->_lineCount;
 279         if ($this->_haveColumnHeader) {
 280           $recordNumber--;
 281         }
 282         array_unshift($values, $recordNumber);
 283         $this->_pledgePaymentErrors[] = $values;
 284       }
 285
 286       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 287         $this->_invalidSoftCreditRowCount++;
 288         $recordNumber = $this->_lineCount;
 289         if ($this->_haveColumnHeader) {
 290           $recordNumber--;
 291         }
 292         array_unshift($values, $recordNumber);
 293         $this->_softCreditErrors[] = $values;
 294       }
 295
 296       if ($returnCode == self::CONFLICT) {
 297         $this->_conflictCount++;
 298         $recordNumber = $this->_lineCount;
 299         if ($this->_haveColumnHeader) {
 300           $recordNumber--;
 301         }
 302         array_unshift($values, $recordNumber);
 303         $this->_conflicts[] = $values;
 304       }
 305
 306       if ($returnCode == self::DUPLICATE) {
 307         if ($returnCode == self::MULTIPLE_DUPE) {
 308           /* TODO: multi-dupes should be counted apart from singles
 309            * on non-skip action */
 310         }
 311         $this->_duplicateCount++;
 312         $recordNumber = $this->_lineCount;
 313         if ($this->_haveColumnHeader) {
 314           $recordNumber--;
 315         }
 316         array_unshift($values, $recordNumber);
 317         $this->_duplicates[] = $values;
 318         if ($onDuplicate != self::DUPLICATE_SKIP) {
 319           $this->_validCount++;
 320         }
 321       }
 322
 323       // we give the derived class a way of aborting the process
 324       // note that the return code could be multiple code or'ed together
 325       if ($returnCode == self::STOP) {
 326         break;
 327       }
 328
 329       // if we are done processing the maxNumber of lines, break
 330       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 331         break;
 332       }
 333     }
 334
 335     fclose($fd);
 336
 337     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 338       $customHeaders = $mapper;
 339
 340       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 341       foreach ($customHeaders as $key => $value) {
 342         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 343           $customHeaders[$key] = $customfields[$id][0];
 344         }
 345       }
 346       if ($this->_invalidRowCount) {
 347         // removed view url for invlaid contacts
 348         $headers = array_merge(array(
 349             ts('Line Number'),
 350             ts('Reason'),
 351           ),
 352           $customHeaders
 353         );
 354         $this->_errorFileName = self::errorFileName(self::ERROR);
 355         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 356       }
 357
 358       if ($this->_invalidPledgePaymentRowCount) {
 359         // removed view url for invlaid contacts
 360         $headers = array_merge(array(
 361             ts('Line Number'),
 362             ts('Reason'),
 363           ),
 364           $customHeaders
 365         );
 366         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 367         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 368       }
 369
 370       if ($this->_invalidSoftCreditRowCount) {
 371         // removed view url for invlaid contacts
 372         $headers = array_merge(array(
 373             ts('Line Number'),
 374             ts('Reason'),
 375           ),
 376           $customHeaders
 377         );
 378         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 379         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 380       }
 381
 382       if ($this->_conflictCount) {
 383         $headers = array_merge(array(
 384             ts('Line Number'),
 385             ts('Reason'),
 386           ),
 387           $customHeaders
 388         );
 389         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 390         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 391       }
 392       if ($this->_duplicateCount) {
 393         $headers = array_merge(array(
 394             ts('Line Number'),
 395             ts('View Contribution URL'),
 396           ),
 397           $customHeaders
 398         );
 399
 400         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 401         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 402       }
 403     }
 404     return $this->fini();
 405   }
 406
 407   /**
 408    * Given a list of the importable field keys that the user has selected
 409    * set the active fields array to this list
 410    *
 411    * @param array $fieldKeys mapped array of values
 412    */
 413   public function setActiveFields($fieldKeys) {
 414     $this->_activeFieldCount = count($fieldKeys);
 415     foreach ($fieldKeys as $key) {
 416       if (empty($this->_fields[$key])) {
 417         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 418       }
 419       else {
 420         $this->_activeFields[] = clone($this->_fields[$key]);
 421       }
 422     }
 423   }
 424
 425   /**
 426    * @param array $elements
 427    */
 428   public function setActiveFieldSoftCredit($elements) {
 429     for ($i = 0; $i < count($elements); $i++) {
 430       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 431     }
 432   }
 433
 434   /**
 435    * @param array $elements
 436    */
 437   public function setActiveFieldSoftCreditType($elements) {
 438     for ($i = 0; $i < count($elements); $i++) {
 439       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 440     }
 441   }
 442
 443   /**
 444    * Format the field values for input to the api.
 445    *
 446    * @return array
 447    *   (reference ) associative array of name/value pairs
 448    */
 449   public function &getActiveFieldParams() {
 450     $params = array();
 451     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 452       if (isset($this->_activeFields[$i]->_value)) {
 453         if (isset($this->_activeFields[$i]->_softCreditField)) {
 454           if (!isset($params[$this->_activeFields[$i]->_name])) {
 455             $params[$this->_activeFields[$i]->_name] = array();
 456           }
 457           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 458           if (isset($this->_activeFields[$i]->_softCreditType)) {
 459             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 460           }
 461         }
 462
 463         if (!isset($params[$this->_activeFields[$i]->_name])) {
 464           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 465             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 466           }
 467         }
 468       }
 469     }
 470     return $params;
 471   }
 472
 473   /**
 474    * @param string $name
 475    * @param $title
 476    * @param int $type
 477    * @param string $headerPattern
 478    * @param string $dataPattern
 479    */
 480   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 481     if (empty($name)) {
 482       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 483     }
 484     else {
 485       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 486       if (!array_key_exists($name, $tempField)) {
 487         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 488       }
 489       else {
 490         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 491           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 492         );
 493       }
 494     }
 495   }
 496
 497   /**
 498    * Store parser values.
 499    *
 500    * @param CRM_Core_Session $store
 501    *
 502    * @param int $mode
 503    */
 504   public function set($store, $mode = self::MODE_SUMMARY) {
 505     $store->set('fileSize', $this->_fileSize);
 506     $store->set('lineCount', $this->_lineCount);
 507     $store->set('seperator', $this->_seperator);
 508     $store->set('fields', $this->getSelectValues());
 509     $store->set('fieldTypes', $this->getSelectTypes());
 510
 511     $store->set('headerPatterns', $this->getHeaderPatterns());
 512     $store->set('dataPatterns', $this->getDataPatterns());
 513     $store->set('columnCount', $this->_activeFieldCount);
 514
 515     $store->set('totalRowCount', $this->_totalCount);
 516     $store->set('validRowCount', $this->_validCount);
 517     $store->set('invalidRowCount', $this->_invalidRowCount);
 518     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 519     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 520     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 521     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 522     $store->set('conflictRowCount', $this->_conflictCount);
 523
 524     switch ($this->_contactType) {
 525       case 'Individual':
 526         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 527         break;
 528
 529       case 'Household':
 530         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 531         break;
 532
 533       case 'Organization':
 534         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 535     }
 536
 537     if ($this->_invalidRowCount) {
 538       $store->set('errorsFileName', $this->_errorFileName);
 539     }
 540     if ($this->_conflictCount) {
 541       $store->set('conflictsFileName', $this->_conflictFileName);
 542     }
 543     if (isset($this->_rows) && !empty($this->_rows)) {
 544       $store->set('dataValues', $this->_rows);
 545     }
 546
 547     if ($this->_invalidPledgePaymentRowCount) {
 548       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 549     }
 550
 551     if ($this->_invalidSoftCreditRowCount) {
 552       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 553     }
 554
 555     if ($mode == self::MODE_IMPORT) {
 556       $store->set('duplicateRowCount', $this->_duplicateCount);
 557       if ($this->_duplicateCount) {
 558         $store->set('duplicatesFileName', $this->_duplicateFileName);
 559       }
 560     }
 561   }
 562
 563   /**
 564    * Export data to a CSV file.
 565    *
 566    * @param string $fileName
 567    * @param array $header
 568    * @param array $data
 569    */
 570   public static function exportCSV($fileName, $header, $data) {
 571     $output = array();
 572     $fd = fopen($fileName, 'w');
 573
 574     foreach ($header as $key => $value) {
 575       $header[$key] = "\"$value\"";
 576     }
 577     $config = CRM_Core_Config::singleton();
 578     $output[] = implode($config->fieldSeparator, $header);
 579
 580     foreach ($data as $datum) {
 581       foreach ($datum as $key => $value) {
 582         if (isset($value[0]) && is_array($value)) {
 583           foreach ($value[0] as $k1 => $v1) {
 584             if ($k1 == 'location_type_id') {
 585               continue;
 586             }
 587             $datum[$k1] = $v1;
 588           }
 589         }
 590         else {
 591           $datum[$key] = "\"$value\"";
 592         }
 593       }
 594       $output[] = implode($config->fieldSeparator, $datum);
 595     }
 596     fwrite($fd, implode("\n", $output));
 597     fclose($fd);
 598   }
 599
 600   /**
 601    * Determines the file extension based on error code.
 602    *
 603    * @param int $type
 604    *   Error code constant.
 605    *
 606    * @return string
 607    */
 608   public static function errorFileName($type) {
 609     $fileName = NULL;
 610     if (empty($type)) {
 611       return $fileName;
 612     }
 613
 614     $config = CRM_Core_Config::singleton();
 615     $fileName = $config->uploadDir . "sqlImport";
 616
 617     switch ($type) {
 618       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 619         $fileName .= '.softCreditErrors';
 620         break;
 621
 622       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 623         $fileName .= '.pledgePaymentErrors';
 624         break;
 625
 626       default:
 627         $fileName = parent::errorFileName($type);
 628         break;
 629     }
 630
 631     return $fileName;
 632   }
 633
 634   /**
 635    * Determines the file name based on error code.
 636    *
 637    * @param int $type
 638    *   Error code constant.
 639    *
 640    * @return string
 641    */
 642   public static function saveFileName($type) {
 643     $fileName = NULL;
 644     if (empty($type)) {
 645       return $fileName;
 646     }
 647
 648     switch ($type) {
 649       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 650         $fileName = 'Import_Soft_Credit_Errors.csv';
 651         break;
 652
 653       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 654         $fileName = 'Import_Pledge_Payment_Errors.csv';
 655         break;
 656
 657       default:
 658         $fileName = parent::saveFileName($type);
 659         break;
 660     }
 661
 662     return $fileName;
 663   }
 664
 665 }