CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  34
  35   /**
  36    * Contribution-specific result codes
  37    * @see CRM_Import_Parser result code constants
  38    */
  39   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  40
  41   /**
  42    * @var string
  43    */
  44   protected $_fileName;
  45
  46   /**
  47    * Imported file size
  48    * @var int
  49    */
  50   protected $_fileSize;
  51
  52   /**
  53    * Seperator being used
  54    * @var string
  55    */
  56   protected $_seperator;
  57
  58   /**
  59    * Total number of lines in file
  60    * @var int
  61    */
  62   protected $_lineCount;
  63
  64   /**
  65    * Running total number of valid soft credit rows
  66    * @var int
  67    */
  68   protected $_validSoftCreditRowCount;
  69
  70   /**
  71    * Running total number of invalid soft credit rows
  72    * @var int
  73    */
  74   protected $_invalidSoftCreditRowCount;
  75
  76   /**
  77    * Running total number of valid pledge payment rows
  78    * @var int
  79    */
  80   protected $_validPledgePaymentRowCount;
  81
  82   /**
  83    * Running total number of invalid pledge payment rows
  84    * @var int
  85    */
  86   protected $_invalidPledgePaymentRowCount;
  87
  88   /**
  89    * Array of pledge payment error lines, bounded by MAX_ERROR
  90    * @var array
  91    */
  92   protected $_pledgePaymentErrors;
  93
  94   /**
  95    * Array of pledge payment error lines, bounded by MAX_ERROR
  96    * @var array
  97    */
  98   protected $_softCreditErrors;
  99
 100   /**
 101    * Filename of pledge payment error data
 102    *
 103    * @var string
 104    */
 105   protected $_pledgePaymentErrorsFileName;
 106
 107   /**
 108    * Filename of soft credit error data
 109    *
 110    * @var string
 111    */
 112   protected $_softCreditErrorsFileName;
 113
 114   /**
 115    * Whether the file has a column header or not
 116    *
 117    * @var bool
 118    */
 119   protected $_haveColumnHeader;
 120
 121   /**
 122    * @param string $fileName
 123    * @param string $seperator
 124    * @param $mapper
 125    * @param bool $skipColumnHeader
 126    * @param int $mode
 127    * @param int $contactType
 128    * @param int $onDuplicate
 129    * @param int $statusID
 130    * @param int $totalRowCount
 131    *
 132    * @return mixed
 133    * @throws Exception
 134    */
 135   public function run(
 136     $fileName,
 137     $seperator = ',',
 138     &$mapper,
 139     $skipColumnHeader = FALSE,
 140     $mode = self::MODE_PREVIEW,
 141     $contactType = self::CONTACT_INDIVIDUAL,
 142     $onDuplicate = self::DUPLICATE_SKIP,
 143     $statusID = NULL,
 144     $totalRowCount = NULL
 145   ) {
 146     if (!is_array($fileName)) {
 147       CRM_Core_Error::fatal();
 148     }
 149     $fileName = $fileName['name'];
 150
 151     switch ($contactType) {
 152       case self::CONTACT_INDIVIDUAL:
 153         $this->_contactType = 'Individual';
 154         break;
 155
 156       case self::CONTACT_HOUSEHOLD:
 157         $this->_contactType = 'Household';
 158         break;
 159
 160       case self::CONTACT_ORGANIZATION:
 161         $this->_contactType = 'Organization';
 162     }
 163
 164     $this->init();
 165
 166     $this->_haveColumnHeader = $skipColumnHeader;
 167
 168     $this->_seperator = $seperator;
 169
 170     $fd = fopen($fileName, "r");
 171     if (!$fd) {
 172       return FALSE;
 173     }
 174
 175     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 176     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 177     $this->_totalCount = $this->_conflictCount = 0;
 178
 179     $this->_errors = [];
 180     $this->_warnings = [];
 181     $this->_conflicts = [];
 182     $this->_pledgePaymentErrors = [];
 183     $this->_softCreditErrors = [];
 184     if ($statusID) {
 185       $this->progressImport($statusID);
 186       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 187     }
 188
 189     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 190
 191     if ($mode == self::MODE_MAPFIELD) {
 192       $this->_rows = [];
 193     }
 194     else {
 195       $this->_activeFieldCount = count($this->_activeFields);
 196     }
 197
 198     while (!feof($fd)) {
 199       $this->_lineCount++;
 200
 201       $values = fgetcsv($fd, 8192, $seperator);
 202       if (!$values) {
 203         continue;
 204       }
 205
 206       self::encloseScrub($values);
 207
 208       // skip column header if we're not in mapfield mode
 209       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 210         $skipColumnHeader = FALSE;
 211         continue;
 212       }
 213
 214       /* trim whitespace around the values */
 215
 216       $empty = TRUE;
 217       foreach ($values as $k => $v) {
 218         $values[$k] = trim($v, " \t\r\n");
 219       }
 220
 221       if (CRM_Utils_System::isNull($values)) {
 222         continue;
 223       }
 224
 225       $this->_totalCount++;
 226
 227       if ($mode == self::MODE_MAPFIELD) {
 228         $returnCode = $this->mapField($values);
 229       }
 230       elseif ($mode == self::MODE_PREVIEW) {
 231         $returnCode = $this->preview($values);
 232       }
 233       elseif ($mode == self::MODE_SUMMARY) {
 234         $returnCode = $this->summary($values);
 235       }
 236       elseif ($mode == self::MODE_IMPORT) {
 237         $returnCode = $this->import($onDuplicate, $values);
 238         if ($statusID && (($this->_lineCount % 50) == 0)) {
 239           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 240         }
 241       }
 242       else {
 243         $returnCode = self::ERROR;
 244       }
 245
 246       // note that a line could be valid but still produce a warning
 247       if ($returnCode == self::VALID) {
 248         $this->_validCount++;
 249         if ($mode == self::MODE_MAPFIELD) {
 250           $this->_rows[] = $values;
 251           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 252         }
 253       }
 254
 255       if ($returnCode == self::SOFT_CREDIT) {
 256         $this->_validSoftCreditRowCount++;
 257         $this->_validCount++;
 258         if ($mode == self::MODE_MAPFIELD) {
 259           $this->_rows[] = $values;
 260           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 261         }
 262       }
 263
 264       if ($returnCode == self::PLEDGE_PAYMENT) {
 265         $this->_validPledgePaymentRowCount++;
 266         $this->_validCount++;
 267         if ($mode == self::MODE_MAPFIELD) {
 268           $this->_rows[] = $values;
 269           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 270         }
 271       }
 272
 273       if ($returnCode == self::WARNING) {
 274         $this->_warningCount++;
 275         if ($this->_warningCount < $this->_maxWarningCount) {
 276           $this->_warningCount[] = $line;
 277         }
 278       }
 279
 280       if ($returnCode == self::ERROR) {
 281         $this->_invalidRowCount++;
 282         $recordNumber = $this->_lineCount;
 283         if ($this->_haveColumnHeader) {
 284           $recordNumber--;
 285         }
 286         array_unshift($values, $recordNumber);
 287         $this->_errors[] = $values;
 288       }
 289
 290       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 291         $this->_invalidPledgePaymentRowCount++;
 292         $recordNumber = $this->_lineCount;
 293         if ($this->_haveColumnHeader) {
 294           $recordNumber--;
 295         }
 296         array_unshift($values, $recordNumber);
 297         $this->_pledgePaymentErrors[] = $values;
 298       }
 299
 300       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 301         $this->_invalidSoftCreditRowCount++;
 302         $recordNumber = $this->_lineCount;
 303         if ($this->_haveColumnHeader) {
 304           $recordNumber--;
 305         }
 306         array_unshift($values, $recordNumber);
 307         $this->_softCreditErrors[] = $values;
 308       }
 309
 310       if ($returnCode == self::CONFLICT) {
 311         $this->_conflictCount++;
 312         $recordNumber = $this->_lineCount;
 313         if ($this->_haveColumnHeader) {
 314           $recordNumber--;
 315         }
 316         array_unshift($values, $recordNumber);
 317         $this->_conflicts[] = $values;
 318       }
 319
 320       if ($returnCode == self::DUPLICATE) {
 321         if ($returnCode == self::MULTIPLE_DUPE) {
 322           /* TODO: multi-dupes should be counted apart from singles
 323            * on non-skip action */
 324         }
 325         $this->_duplicateCount++;
 326         $recordNumber = $this->_lineCount;
 327         if ($this->_haveColumnHeader) {
 328           $recordNumber--;
 329         }
 330         array_unshift($values, $recordNumber);
 331         $this->_duplicates[] = $values;
 332         if ($onDuplicate != self::DUPLICATE_SKIP) {
 333           $this->_validCount++;
 334         }
 335       }
 336
 337       // we give the derived class a way of aborting the process
 338       // note that the return code could be multiple code or'ed together
 339       if ($returnCode == self::STOP) {
 340         break;
 341       }
 342
 343       // if we are done processing the maxNumber of lines, break
 344       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 345         break;
 346       }
 347     }
 348
 349     fclose($fd);
 350
 351     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 352       $customHeaders = $mapper;
 353
 354       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 355       foreach ($customHeaders as $key => $value) {
 356         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 357           $customHeaders[$key] = $customfields[$id][0];
 358         }
 359       }
 360       if ($this->_invalidRowCount) {
 361         // removed view url for invlaid contacts
 362         $headers = array_merge([
 363           ts('Line Number'),
 364           ts('Reason'),
 365         ], $customHeaders);
 366         $this->_errorFileName = self::errorFileName(self::ERROR);
 367         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 368       }
 369
 370       if ($this->_invalidPledgePaymentRowCount) {
 371         // removed view url for invlaid contacts
 372         $headers = array_merge([
 373           ts('Line Number'),
 374           ts('Reason'),
 375         ], $customHeaders);
 376         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 377         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 378       }
 379
 380       if ($this->_invalidSoftCreditRowCount) {
 381         // removed view url for invlaid contacts
 382         $headers = array_merge([
 383           ts('Line Number'),
 384           ts('Reason'),
 385         ], $customHeaders);
 386         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 387         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 388       }
 389
 390       if ($this->_conflictCount) {
 391         $headers = array_merge([
 392           ts('Line Number'),
 393           ts('Reason'),
 394         ], $customHeaders);
 395         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 396         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 397       }
 398       if ($this->_duplicateCount) {
 399         $headers = array_merge([
 400           ts('Line Number'),
 401           ts('View Contribution URL'),
 402         ], $customHeaders);
 403
 404         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 405         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 406       }
 407     }
 408     return $this->fini();
 409   }
 410
 411   /**
 412    * Given a list of the importable field keys that the user has selected
 413    * set the active fields array to this list
 414    *
 415    * @param array $fieldKeys mapped array of values
 416    */
 417   public function setActiveFields($fieldKeys) {
 418     $this->_activeFieldCount = count($fieldKeys);
 419     foreach ($fieldKeys as $key) {
 420       if (empty($this->_fields[$key])) {
 421         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 422       }
 423       else {
 424         $this->_activeFields[] = clone($this->_fields[$key]);
 425       }
 426     }
 427   }
 428
 429   /**
 430    * @param array $elements
 431    */
 432   public function setActiveFieldSoftCredit($elements) {
 433     for ($i = 0; $i < count($elements); $i++) {
 434       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 435     }
 436   }
 437
 438   /**
 439    * @param array $elements
 440    */
 441   public function setActiveFieldSoftCreditType($elements) {
 442     for ($i = 0; $i < count($elements); $i++) {
 443       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 444     }
 445   }
 446
 447   /**
 448    * Format the field values for input to the api.
 449    *
 450    * @return array
 451    *   (reference ) associative array of name/value pairs
 452    */
 453   public function &getActiveFieldParams() {
 454     $params = [];
 455     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 456       if (isset($this->_activeFields[$i]->_value)) {
 457         if (isset($this->_activeFields[$i]->_softCreditField)) {
 458           if (!isset($params[$this->_activeFields[$i]->_name])) {
 459             $params[$this->_activeFields[$i]->_name] = [];
 460           }
 461           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 462           if (isset($this->_activeFields[$i]->_softCreditType)) {
 463             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 464           }
 465         }
 466
 467         if (!isset($params[$this->_activeFields[$i]->_name])) {
 468           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 469             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 470           }
 471         }
 472       }
 473     }
 474     return $params;
 475   }
 476
 477   /**
 478    * @param string $name
 479    * @param $title
 480    * @param int $type
 481    * @param string $headerPattern
 482    * @param string $dataPattern
 483    */
 484   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 485     if (empty($name)) {
 486       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 487     }
 488     else {
 489       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 490       if (!array_key_exists($name, $tempField)) {
 491         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 492       }
 493       else {
 494         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 495           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 496         );
 497       }
 498     }
 499   }
 500
 501   /**
 502    * Store parser values.
 503    *
 504    * @param CRM_Core_Session $store
 505    *
 506    * @param int $mode
 507    */
 508   public function set($store, $mode = self::MODE_SUMMARY) {
 509     $store->set('fileSize', $this->_fileSize);
 510     $store->set('lineCount', $this->_lineCount);
 511     $store->set('seperator', $this->_seperator);
 512     $store->set('fields', $this->getSelectValues());
 513     $store->set('fieldTypes', $this->getSelectTypes());
 514
 515     $store->set('headerPatterns', $this->getHeaderPatterns());
 516     $store->set('dataPatterns', $this->getDataPatterns());
 517     $store->set('columnCount', $this->_activeFieldCount);
 518
 519     $store->set('totalRowCount', $this->_totalCount);
 520     $store->set('validRowCount', $this->_validCount);
 521     $store->set('invalidRowCount', $this->_invalidRowCount);
 522     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 523     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 524     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 525     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 526     $store->set('conflictRowCount', $this->_conflictCount);
 527
 528     switch ($this->_contactType) {
 529       case 'Individual':
 530         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 531         break;
 532
 533       case 'Household':
 534         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 535         break;
 536
 537       case 'Organization':
 538         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 539     }
 540
 541     if ($this->_invalidRowCount) {
 542       $store->set('errorsFileName', $this->_errorFileName);
 543     }
 544     if ($this->_conflictCount) {
 545       $store->set('conflictsFileName', $this->_conflictFileName);
 546     }
 547     if (isset($this->_rows) && !empty($this->_rows)) {
 548       $store->set('dataValues', $this->_rows);
 549     }
 550
 551     if ($this->_invalidPledgePaymentRowCount) {
 552       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 553     }
 554
 555     if ($this->_invalidSoftCreditRowCount) {
 556       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 557     }
 558
 559     if ($mode == self::MODE_IMPORT) {
 560       $store->set('duplicateRowCount', $this->_duplicateCount);
 561       if ($this->_duplicateCount) {
 562         $store->set('duplicatesFileName', $this->_duplicateFileName);
 563       }
 564     }
 565   }
 566
 567   /**
 568    * Export data to a CSV file.
 569    *
 570    * @param string $fileName
 571    * @param array $header
 572    * @param array $data
 573    */
 574   public static function exportCSV($fileName, $header, $data) {
 575     $output = [];
 576     $fd = fopen($fileName, 'w');
 577
 578     foreach ($header as $key => $value) {
 579       $header[$key] = "\"$value\"";
 580     }
 581     $config = CRM_Core_Config::singleton();
 582     $output[] = implode($config->fieldSeparator, $header);
 583
 584     foreach ($data as $datum) {
 585       foreach ($datum as $key => $value) {
 586         if (isset($value[0]) && is_array($value)) {
 587           foreach ($value[0] as $k1 => $v1) {
 588             if ($k1 == 'location_type_id') {
 589               continue;
 590             }
 591             $datum[$k1] = $v1;
 592           }
 593         }
 594         else {
 595           $datum[$key] = "\"$value\"";
 596         }
 597       }
 598       $output[] = implode($config->fieldSeparator, $datum);
 599     }
 600     fwrite($fd, implode("\n", $output));
 601     fclose($fd);
 602   }
 603
 604   /**
 605    * Determines the file extension based on error code.
 606    *
 607    * @param int $type
 608    *   Error code constant.
 609    *
 610    * @return string
 611    */
 612   public static function errorFileName($type) {
 613     $fileName = NULL;
 614     if (empty($type)) {
 615       return $fileName;
 616     }
 617
 618     $config = CRM_Core_Config::singleton();
 619     $fileName = $config->uploadDir . "sqlImport";
 620
 621     switch ($type) {
 622       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 623         $fileName .= '.softCreditErrors';
 624         break;
 625
 626       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 627         $fileName .= '.pledgePaymentErrors';
 628         break;
 629
 630       default:
 631         $fileName = parent::errorFileName($type);
 632         break;
 633     }
 634
 635     return $fileName;
 636   }
 637
 638   /**
 639    * Determines the file name based on error code.
 640    *
 641    * @param int $type
 642    *   Error code constant.
 643    *
 644    * @return string
 645    */
 646   public static function saveFileName($type) {
 647     $fileName = NULL;
 648     if (empty($type)) {
 649       return $fileName;
 650     }
 651
 652     switch ($type) {
 653       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 654         $fileName = 'Import_Soft_Credit_Errors.csv';
 655         break;
 656
 657       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 658         $fileName = 'Import_Pledge_Payment_Errors.csv';
 659         break;
 660
 661       default:
 662         $fileName = parent::saveFileName($type);
 663         break;
 664     }
 665
 666     return $fileName;
 667   }
 668
 669 }