CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  18
  19   /**
  20    * Contribution-specific result codes
  21    * @see CRM_Import_Parser result code constants
  22    */
  23   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  24
  25   /**
  26    * @var string
  27    */
  28   protected $_fileName;
  29
  30   /**
  31    * Imported file size
  32    * @var int
  33    */
  34   protected $_fileSize;
  35
  36   /**
  37    * Seperator being used
  38    * @var string
  39    */
  40   protected $_seperator;
  41
  42   /**
  43    * Total number of lines in file
  44    * @var int
  45    */
  46   protected $_lineCount;
  47
  48   /**
  49    * Running total number of valid soft credit rows
  50    * @var int
  51    */
  52   protected $_validSoftCreditRowCount;
  53
  54   /**
  55    * Running total number of invalid soft credit rows
  56    * @var int
  57    */
  58   protected $_invalidSoftCreditRowCount;
  59
  60   /**
  61    * Running total number of valid pledge payment rows
  62    * @var int
  63    */
  64   protected $_validPledgePaymentRowCount;
  65
  66   /**
  67    * Running total number of invalid pledge payment rows
  68    * @var int
  69    */
  70   protected $_invalidPledgePaymentRowCount;
  71
  72   /**
  73    * Array of pledge payment error lines, bounded by MAX_ERROR
  74    * @var array
  75    */
  76   protected $_pledgePaymentErrors;
  77
  78   /**
  79    * Array of pledge payment error lines, bounded by MAX_ERROR
  80    * @var array
  81    */
  82   protected $_softCreditErrors;
  83
  84   /**
  85    * Filename of pledge payment error data
  86    *
  87    * @var string
  88    */
  89   protected $_pledgePaymentErrorsFileName;
  90
  91   /**
  92    * Filename of soft credit error data
  93    *
  94    * @var string
  95    */
  96   protected $_softCreditErrorsFileName;
  97
  98   /**
  99    * Whether the file has a column header or not
 100    *
 101    * @var bool
 102    */
 103   protected $_haveColumnHeader;
 104
 105   /**
 106    * @param string $fileName
 107    * @param string $seperator
 108    * @param $mapper
 109    * @param bool $skipColumnHeader
 110    * @param int $mode
 111    * @param int $contactType
 112    * @param int $onDuplicate
 113    * @param int $statusID
 114    * @param int $totalRowCount
 115    *
 116    * @return mixed
 117    * @throws Exception
 118    */
 119   public function run(
 120     $fileName,
 121     $seperator = ',',
 122     &$mapper,
 123     $skipColumnHeader = FALSE,
 124     $mode = self::MODE_PREVIEW,
 125     $contactType = self::CONTACT_INDIVIDUAL,
 126     $onDuplicate = self::DUPLICATE_SKIP,
 127     $statusID = NULL,
 128     $totalRowCount = NULL
 129   ) {
 130     if (!is_array($fileName)) {
 131       CRM_Core_Error::fatal();
 132     }
 133     $fileName = $fileName['name'];
 134
 135     switch ($contactType) {
 136       case self::CONTACT_INDIVIDUAL:
 137         $this->_contactType = 'Individual';
 138         break;
 139
 140       case self::CONTACT_HOUSEHOLD:
 141         $this->_contactType = 'Household';
 142         break;
 143
 144       case self::CONTACT_ORGANIZATION:
 145         $this->_contactType = 'Organization';
 146     }
 147
 148     $this->init();
 149
 150     $this->_haveColumnHeader = $skipColumnHeader;
 151
 152     $this->_seperator = $seperator;
 153
 154     $fd = fopen($fileName, "r");
 155     if (!$fd) {
 156       return FALSE;
 157     }
 158
 159     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 160     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 161     $this->_totalCount = $this->_conflictCount = 0;
 162
 163     $this->_errors = [];
 164     $this->_warnings = [];
 165     $this->_conflicts = [];
 166     $this->_pledgePaymentErrors = [];
 167     $this->_softCreditErrors = [];
 168     if ($statusID) {
 169       $this->progressImport($statusID);
 170       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 171     }
 172
 173     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 174
 175     if ($mode == self::MODE_MAPFIELD) {
 176       $this->_rows = [];
 177     }
 178     else {
 179       $this->_activeFieldCount = count($this->_activeFields);
 180     }
 181
 182     while (!feof($fd)) {
 183       $this->_lineCount++;
 184
 185       $values = fgetcsv($fd, 8192, $seperator);
 186       if (!$values) {
 187         continue;
 188       }
 189
 190       self::encloseScrub($values);
 191
 192       // skip column header if we're not in mapfield mode
 193       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 194         $skipColumnHeader = FALSE;
 195         continue;
 196       }
 197
 198       /* trim whitespace around the values */
 199
 200       $empty = TRUE;
 201       foreach ($values as $k => $v) {
 202         $values[$k] = trim($v, " \t\r\n");
 203       }
 204
 205       if (CRM_Utils_System::isNull($values)) {
 206         continue;
 207       }
 208
 209       $this->_totalCount++;
 210
 211       if ($mode == self::MODE_MAPFIELD) {
 212         $returnCode = $this->mapField($values);
 213       }
 214       elseif ($mode == self::MODE_PREVIEW) {
 215         $returnCode = $this->preview($values);
 216       }
 217       elseif ($mode == self::MODE_SUMMARY) {
 218         $returnCode = $this->summary($values);
 219       }
 220       elseif ($mode == self::MODE_IMPORT) {
 221         $returnCode = $this->import($onDuplicate, $values);
 222         if ($statusID && (($this->_lineCount % 50) == 0)) {
 223           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 224         }
 225       }
 226       else {
 227         $returnCode = self::ERROR;
 228       }
 229
 230       // note that a line could be valid but still produce a warning
 231       if ($returnCode == self::VALID) {
 232         $this->_validCount++;
 233         if ($mode == self::MODE_MAPFIELD) {
 234           $this->_rows[] = $values;
 235           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 236         }
 237       }
 238
 239       if ($returnCode == self::SOFT_CREDIT) {
 240         $this->_validSoftCreditRowCount++;
 241         $this->_validCount++;
 242         if ($mode == self::MODE_MAPFIELD) {
 243           $this->_rows[] = $values;
 244           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 245         }
 246       }
 247
 248       if ($returnCode == self::PLEDGE_PAYMENT) {
 249         $this->_validPledgePaymentRowCount++;
 250         $this->_validCount++;
 251         if ($mode == self::MODE_MAPFIELD) {
 252           $this->_rows[] = $values;
 253           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 254         }
 255       }
 256
 257       if ($returnCode == self::WARNING) {
 258         $this->_warningCount++;
 259         if ($this->_warningCount < $this->_maxWarningCount) {
 260           $this->_warningCount[] = $line;
 261         }
 262       }
 263
 264       if ($returnCode == self::ERROR) {
 265         $this->_invalidRowCount++;
 266         $recordNumber = $this->_lineCount;
 267         if ($this->_haveColumnHeader) {
 268           $recordNumber--;
 269         }
 270         array_unshift($values, $recordNumber);
 271         $this->_errors[] = $values;
 272       }
 273
 274       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 275         $this->_invalidPledgePaymentRowCount++;
 276         $recordNumber = $this->_lineCount;
 277         if ($this->_haveColumnHeader) {
 278           $recordNumber--;
 279         }
 280         array_unshift($values, $recordNumber);
 281         $this->_pledgePaymentErrors[] = $values;
 282       }
 283
 284       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 285         $this->_invalidSoftCreditRowCount++;
 286         $recordNumber = $this->_lineCount;
 287         if ($this->_haveColumnHeader) {
 288           $recordNumber--;
 289         }
 290         array_unshift($values, $recordNumber);
 291         $this->_softCreditErrors[] = $values;
 292       }
 293
 294       if ($returnCode == self::CONFLICT) {
 295         $this->_conflictCount++;
 296         $recordNumber = $this->_lineCount;
 297         if ($this->_haveColumnHeader) {
 298           $recordNumber--;
 299         }
 300         array_unshift($values, $recordNumber);
 301         $this->_conflicts[] = $values;
 302       }
 303
 304       if ($returnCode == self::DUPLICATE) {
 305         if ($returnCode == self::MULTIPLE_DUPE) {
 306           /* TODO: multi-dupes should be counted apart from singles
 307            * on non-skip action */
 308         }
 309         $this->_duplicateCount++;
 310         $recordNumber = $this->_lineCount;
 311         if ($this->_haveColumnHeader) {
 312           $recordNumber--;
 313         }
 314         array_unshift($values, $recordNumber);
 315         $this->_duplicates[] = $values;
 316         if ($onDuplicate != self::DUPLICATE_SKIP) {
 317           $this->_validCount++;
 318         }
 319       }
 320
 321       // we give the derived class a way of aborting the process
 322       // note that the return code could be multiple code or'ed together
 323       if ($returnCode == self::STOP) {
 324         break;
 325       }
 326
 327       // if we are done processing the maxNumber of lines, break
 328       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 329         break;
 330       }
 331     }
 332
 333     fclose($fd);
 334
 335     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 336       $customHeaders = $mapper;
 337
 338       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 339       foreach ($customHeaders as $key => $value) {
 340         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 341           $customHeaders[$key] = $customfields[$id][0];
 342         }
 343       }
 344       if ($this->_invalidRowCount) {
 345         // removed view url for invlaid contacts
 346         $headers = array_merge([
 347           ts('Line Number'),
 348           ts('Reason'),
 349         ], $customHeaders);
 350         $this->_errorFileName = self::errorFileName(self::ERROR);
 351         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 352       }
 353
 354       if ($this->_invalidPledgePaymentRowCount) {
 355         // removed view url for invlaid contacts
 356         $headers = array_merge([
 357           ts('Line Number'),
 358           ts('Reason'),
 359         ], $customHeaders);
 360         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 361         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 362       }
 363
 364       if ($this->_invalidSoftCreditRowCount) {
 365         // removed view url for invlaid contacts
 366         $headers = array_merge([
 367           ts('Line Number'),
 368           ts('Reason'),
 369         ], $customHeaders);
 370         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 371         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 372       }
 373
 374       if ($this->_conflictCount) {
 375         $headers = array_merge([
 376           ts('Line Number'),
 377           ts('Reason'),
 378         ], $customHeaders);
 379         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 380         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 381       }
 382       if ($this->_duplicateCount) {
 383         $headers = array_merge([
 384           ts('Line Number'),
 385           ts('View Contribution URL'),
 386         ], $customHeaders);
 387
 388         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 389         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 390       }
 391     }
 392     return $this->fini();
 393   }
 394
 395   /**
 396    * Given a list of the importable field keys that the user has selected
 397    * set the active fields array to this list
 398    *
 399    * @param array $fieldKeys mapped array of values
 400    */
 401   public function setActiveFields($fieldKeys) {
 402     $this->_activeFieldCount = count($fieldKeys);
 403     foreach ($fieldKeys as $key) {
 404       if (empty($this->_fields[$key])) {
 405         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 406       }
 407       else {
 408         $this->_activeFields[] = clone($this->_fields[$key]);
 409       }
 410     }
 411   }
 412
 413   /**
 414    * Store the soft credit field information.
 415    *
 416    * This  was perhaps done this way on the believe that a lot of code pain
 417    * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise
 418    * readability & maintainability next since we can just work with functions to retrieve
 419    * data from the metadata.
 420    *
 421    * @param array $elements
 422    */
 423   public function setActiveFieldSoftCredit($elements) {
 424     foreach ((array) $elements as $i => $element) {
 425       $this->_activeFields[$i]->_softCreditField = $element;
 426     }
 427   }
 428
 429   /**
 430    * Store the soft credit field type information.
 431    *
 432    * This  was perhaps done this way on the believe that a lot of code pain
 433    * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise
 434    * readability & maintainability next since we can just work with functions to retrieve
 435    * data from the metadata.
 436    *
 437    * @param array $elements
 438    */
 439   public function setActiveFieldSoftCreditType($elements) {
 440     foreach ((array) $elements as $i => $element) {
 441       $this->_activeFields[$i]->_softCreditType = $element;
 442     }
 443   }
 444
 445   /**
 446    * Format the field values for input to the api.
 447    *
 448    * @return array
 449    *   (reference ) associative array of name/value pairs
 450    */
 451   public function &getActiveFieldParams() {
 452     $params = [];
 453     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 454       if (isset($this->_activeFields[$i]->_value)) {
 455         if (isset($this->_activeFields[$i]->_softCreditField)) {
 456           if (!isset($params[$this->_activeFields[$i]->_name])) {
 457             $params[$this->_activeFields[$i]->_name] = [];
 458           }
 459           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 460           if (isset($this->_activeFields[$i]->_softCreditType)) {
 461             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 462           }
 463         }
 464
 465         if (!isset($params[$this->_activeFields[$i]->_name])) {
 466           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 467             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 468           }
 469         }
 470       }
 471     }
 472     return $params;
 473   }
 474
 475   /**
 476    * @param string $name
 477    * @param $title
 478    * @param int $type
 479    * @param string $headerPattern
 480    * @param string $dataPattern
 481    */
 482   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 483     if (empty($name)) {
 484       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 485     }
 486     else {
 487       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 488       if (!array_key_exists($name, $tempField)) {
 489         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 490       }
 491       else {
 492         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 493           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 494         );
 495       }
 496     }
 497   }
 498
 499   /**
 500    * Store parser values.
 501    *
 502    * @param CRM_Core_Session $store
 503    *
 504    * @param int $mode
 505    */
 506   public function set($store, $mode = self::MODE_SUMMARY) {
 507     $store->set('fileSize', $this->_fileSize);
 508     $store->set('lineCount', $this->_lineCount);
 509     $store->set('seperator', $this->_seperator);
 510     $store->set('fields', $this->getSelectValues());
 511     $store->set('fieldTypes', $this->getSelectTypes());
 512
 513     $store->set('headerPatterns', $this->getHeaderPatterns());
 514     $store->set('dataPatterns', $this->getDataPatterns());
 515     $store->set('columnCount', $this->_activeFieldCount);
 516
 517     $store->set('totalRowCount', $this->_totalCount);
 518     $store->set('validRowCount', $this->_validCount);
 519     $store->set('invalidRowCount', $this->_invalidRowCount);
 520     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 521     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 522     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 523     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 524     $store->set('conflictRowCount', $this->_conflictCount);
 525
 526     switch ($this->_contactType) {
 527       case 'Individual':
 528         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 529         break;
 530
 531       case 'Household':
 532         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 533         break;
 534
 535       case 'Organization':
 536         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 537     }
 538
 539     if ($this->_invalidRowCount) {
 540       $store->set('errorsFileName', $this->_errorFileName);
 541     }
 542     if ($this->_conflictCount) {
 543       $store->set('conflictsFileName', $this->_conflictFileName);
 544     }
 545     if (isset($this->_rows) && !empty($this->_rows)) {
 546       $store->set('dataValues', $this->_rows);
 547     }
 548
 549     if ($this->_invalidPledgePaymentRowCount) {
 550       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 551     }
 552
 553     if ($this->_invalidSoftCreditRowCount) {
 554       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 555     }
 556
 557     if ($mode == self::MODE_IMPORT) {
 558       $store->set('duplicateRowCount', $this->_duplicateCount);
 559       if ($this->_duplicateCount) {
 560         $store->set('duplicatesFileName', $this->_duplicateFileName);
 561       }
 562     }
 563   }
 564
 565   /**
 566    * Export data to a CSV file.
 567    *
 568    * @param string $fileName
 569    * @param array $header
 570    * @param array $data
 571    */
 572   public static function exportCSV($fileName, $header, $data) {
 573     $output = [];
 574     $fd = fopen($fileName, 'w');
 575
 576     foreach ($header as $key => $value) {
 577       $header[$key] = "\"$value\"";
 578     }
 579     $config = CRM_Core_Config::singleton();
 580     $output[] = implode($config->fieldSeparator, $header);
 581
 582     foreach ($data as $datum) {
 583       foreach ($datum as $key => $value) {
 584         if (isset($value[0]) && is_array($value)) {
 585           foreach ($value[0] as $k1 => $v1) {
 586             if ($k1 == 'location_type_id') {
 587               continue;
 588             }
 589             $datum[$k1] = $v1;
 590           }
 591         }
 592         else {
 593           $datum[$key] = "\"$value\"";
 594         }
 595       }
 596       $output[] = implode($config->fieldSeparator, $datum);
 597     }
 598     fwrite($fd, implode("\n", $output));
 599     fclose($fd);
 600   }
 601
 602   /**
 603    * Determines the file extension based on error code.
 604    *
 605    * @param int $type
 606    *   Error code constant.
 607    *
 608    * @return string
 609    */
 610   public static function errorFileName($type) {
 611     $fileName = NULL;
 612     if (empty($type)) {
 613       return $fileName;
 614     }
 615
 616     $config = CRM_Core_Config::singleton();
 617     $fileName = $config->uploadDir . "sqlImport";
 618
 619     switch ($type) {
 620       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 621         $fileName .= '.softCreditErrors';
 622         break;
 623
 624       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 625         $fileName .= '.pledgePaymentErrors';
 626         break;
 627
 628       default:
 629         $fileName = parent::errorFileName($type);
 630         break;
 631     }
 632
 633     return $fileName;
 634   }
 635
 636   /**
 637    * Determines the file name based on error code.
 638    *
 639    * @param int $type
 640    *   Error code constant.
 641    *
 642    * @return string
 643    */
 644   public static function saveFileName($type) {
 645     $fileName = NULL;
 646     if (empty($type)) {
 647       return $fileName;
 648     }
 649
 650     switch ($type) {
 651       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 652         $fileName = 'Import_Soft_Credit_Errors.csv';
 653         break;
 654
 655       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 656         $fileName = 'Import_Pledge_Payment_Errors.csv';
 657         break;
 658
 659       default:
 660         $fileName = parent::saveFileName($type);
 661         break;
 662     }
 663
 664     return $fileName;
 665   }
 666
 667 }