CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.6                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  37
  38   /**
  39    * Contribution-specific result codes
  40    * @see CRM_Import_Parser result code constants
  41    */
  42   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  43
  44   protected $_fileName;
  45
  46   /**
  47    * Imported file size
  48    */
  49   protected $_fileSize;
  50
  51   /**
  52    * Seperator being used
  53    */
  54   protected $_seperator;
  55
  56   /**
  57    * Total number of lines in file
  58    */
  59   protected $_lineCount;
  60
  61   /**
  62    * Running total number of valid soft credit rows
  63    */
  64   protected $_validSoftCreditRowCount;
  65
  66   /**
  67    * Running total number of invalid soft credit rows
  68    */
  69   protected $_invalidSoftCreditRowCount;
  70
  71   /**
  72    * Running total number of valid pledge payment rows
  73    */
  74   protected $_validPledgePaymentRowCount;
  75
  76   /**
  77    * Running total number of invalid pledge payment rows
  78    */
  79   protected $_invalidPledgePaymentRowCount;
  80
  81   /**
  82    * Array of pledge payment error lines, bounded by MAX_ERROR
  83    */
  84   protected $_pledgePaymentErrors;
  85
  86   /**
  87    * Array of pledge payment error lines, bounded by MAX_ERROR
  88    */
  89   protected $_softCreditErrors;
  90
  91   /**
  92    * Filename of pledge payment error data
  93    *
  94    * @var string
  95    */
  96   protected $_pledgePaymentErrorsFileName;
  97
  98   /**
  99    * Filename of soft credit error data
 100    *
 101    * @var string
 102    */
 103   protected $_softCreditErrorsFileName;
 104
 105   /**
 106    * Whether the file has a column header or not
 107    *
 108    * @var boolean
 109    */
 110   protected $_haveColumnHeader;
 111
 112   /**
 113    * @param string $fileName
 114    * @param string $seperator
 115    * @param $mapper
 116    * @param bool $skipColumnHeader
 117    * @param int $mode
 118    * @param int $contactType
 119    * @param int $onDuplicate
 120    *
 121    * @return mixed
 122    * @throws Exception
 123    */
 124   function run(
 125     $fileName,
 126     $seperator = ',',
 127     &$mapper,
 128     $skipColumnHeader = FALSE,
 129     $mode = self::MODE_PREVIEW,
 130     $contactType = self::CONTACT_INDIVIDUAL,
 131     $onDuplicate = self::DUPLICATE_SKIP
 132   ) {
 133     if (!is_array($fileName)) {
 134       CRM_Core_Error::fatal();
 135     }
 136     $fileName = $fileName['name'];
 137
 138     switch ($contactType) {
 139       case self::CONTACT_INDIVIDUAL:
 140         $this->_contactType = 'Individual';
 141         break;
 142
 143       case self::CONTACT_HOUSEHOLD:
 144         $this->_contactType = 'Household';
 145         break;
 146
 147       case self::CONTACT_ORGANIZATION:
 148         $this->_contactType = 'Organization';
 149     }
 150
 151     $this->init();
 152
 153     $this->_haveColumnHeader = $skipColumnHeader;
 154
 155     $this->_seperator = $seperator;
 156
 157     $fd = fopen($fileName, "r");
 158     if (!$fd) {
 159       return FALSE;
 160     }
 161
 162     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 163     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 164     $this->_totalCount = $this->_conflictCount = 0;
 165
 166     $this->_errors = array();
 167     $this->_warnings = array();
 168     $this->_conflicts = array();
 169     $this->_pledgePaymentErrors = array();
 170     $this->_softCreditErrors = array();
 171
 172     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 173
 174     if ($mode == self::MODE_MAPFIELD) {
 175       $this->_rows = array();
 176     }
 177     else {
 178       $this->_activeFieldCount = count($this->_activeFields);
 179     }
 180
 181     while (!feof($fd)) {
 182       $this->_lineCount++;
 183
 184       $values = fgetcsv($fd, 8192, $seperator);
 185       if (!$values) {
 186         continue;
 187       }
 188
 189       self::encloseScrub($values);
 190
 191       // skip column header if we're not in mapfield mode
 192       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 193         $skipColumnHeader = FALSE;
 194         continue;
 195       }
 196
 197       /* trim whitespace around the values */
 198
 199       $empty = TRUE;
 200       foreach ($values as $k => $v) {
 201         $values[$k] = trim($v, " \t\r\n");
 202       }
 203
 204       if (CRM_Utils_System::isNull($values)) {
 205         continue;
 206       }
 207
 208       $this->_totalCount++;
 209
 210       if ($mode == self::MODE_MAPFIELD) {
 211         $returnCode = $this->mapField($values);
 212       }
 213       elseif ($mode == self::MODE_PREVIEW) {
 214         $returnCode = $this->preview($values);
 215       }
 216       elseif ($mode == self::MODE_SUMMARY) {
 217         $returnCode = $this->summary($values);
 218       }
 219       elseif ($mode == self::MODE_IMPORT) {
 220         $returnCode = $this->import($onDuplicate, $values);
 221       }
 222       else {
 223         $returnCode = self::ERROR;
 224       }
 225
 226       // note that a line could be valid but still produce a warning
 227       if ($returnCode == self::VALID) {
 228         $this->_validCount++;
 229         if ($mode == self::MODE_MAPFIELD) {
 230           $this->_rows[] = $values;
 231           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 232         }
 233       }
 234
 235       if ($returnCode == self::SOFT_CREDIT) {
 236         $this->_validSoftCreditRowCount++;
 237         $this->_validCount++;
 238         if ($mode == self::MODE_MAPFIELD) {
 239           $this->_rows[] = $values;
 240           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 241         }
 242       }
 243
 244       if ($returnCode == self::PLEDGE_PAYMENT) {
 245         $this->_validPledgePaymentRowCount++;
 246         $this->_validCount++;
 247         if ($mode == self::MODE_MAPFIELD) {
 248           $this->_rows[] = $values;
 249           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 250         }
 251       }
 252
 253       if ($returnCode == self::WARNING) {
 254         $this->_warningCount++;
 255         if ($this->_warningCount < $this->_maxWarningCount) {
 256           $this->_warningCount[] = $line;
 257         }
 258       }
 259
 260       if ($returnCode == self::ERROR) {
 261         $this->_invalidRowCount++;
 262         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 263           $recordNumber = $this->_lineCount;
 264           if ($this->_haveColumnHeader) {
 265             $recordNumber--;
 266           }
 267           array_unshift($values, $recordNumber);
 268           $this->_errors[] = $values;
 269         }
 270       }
 271
 272       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 273         $this->_invalidPledgePaymentRowCount++;
 274         if ($this->_invalidPledgePaymentRowCount < $this->_maxErrorCount) {
 275           $recordNumber = $this->_lineCount;
 276           if ($this->_haveColumnHeader) {
 277             $recordNumber--;
 278           }
 279           array_unshift($values, $recordNumber);
 280           $this->_pledgePaymentErrors[] = $values;
 281         }
 282       }
 283
 284       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 285         $this->_invalidSoftCreditRowCount++;
 286         if ($this->_invalidSoftCreditRowCount < $this->_maxErrorCount) {
 287           $recordNumber = $this->_lineCount;
 288           if ($this->_haveColumnHeader) {
 289             $recordNumber--;
 290           }
 291           array_unshift($values, $recordNumber);
 292           $this->_softCreditErrors[] = $values;
 293         }
 294       }
 295
 296       if ($returnCode == self::CONFLICT) {
 297         $this->_conflictCount++;
 298         $recordNumber = $this->_lineCount;
 299         if ($this->_haveColumnHeader) {
 300           $recordNumber--;
 301         }
 302         array_unshift($values, $recordNumber);
 303         $this->_conflicts[] = $values;
 304       }
 305
 306       if ($returnCode == self::DUPLICATE) {
 307         if ($returnCode == self::MULTIPLE_DUPE) {
 308           /* TODO: multi-dupes should be counted apart from singles
 309                      * on non-skip action */
 310         }
 311         $this->_duplicateCount++;
 312         $recordNumber = $this->_lineCount;
 313         if ($this->_haveColumnHeader) {
 314           $recordNumber--;
 315         }
 316         array_unshift($values, $recordNumber);
 317         $this->_duplicates[] = $values;
 318         if ($onDuplicate != self::DUPLICATE_SKIP) {
 319           $this->_validCount++;
 320         }
 321       }
 322
 323       // we give the derived class a way of aborting the process
 324       // note that the return code could be multiple code or'ed together
 325       if ($returnCode == self::STOP) {
 326         break;
 327       }
 328
 329       // if we are done processing the maxNumber of lines, break
 330       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 331         break;
 332       }
 333     }
 334
 335     fclose($fd);
 336
 337     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 338       $customHeaders = $mapper;
 339
 340       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 341       foreach ($customHeaders as $key => $value) {
 342         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 343           $customHeaders[$key] = $customfields[$id][0];
 344         }
 345       }
 346       if ($this->_invalidRowCount) {
 347         // removed view url for invlaid contacts
 348         $headers = array_merge(array(ts('Line Number'),
 349             ts('Reason'),
 350           ),
 351           $customHeaders
 352         );
 353         $this->_errorFileName = self::errorFileName(self::ERROR);
 354         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 355       }
 356
 357       if ($this->_invalidPledgePaymentRowCount) {
 358         // removed view url for invlaid contacts
 359         $headers = array_merge(array(ts('Line Number'),
 360             ts('Reason'),
 361           ),
 362           $customHeaders
 363         );
 364         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 365         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 366       }
 367
 368       if ($this->_invalidSoftCreditRowCount) {
 369         // removed view url for invlaid contacts
 370         $headers = array_merge(array(ts('Line Number'),
 371             ts('Reason'),
 372           ),
 373           $customHeaders
 374         );
 375         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 376         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 377       }
 378
 379       if ($this->_conflictCount) {
 380         $headers = array_merge(array(ts('Line Number'),
 381             ts('Reason'),
 382           ),
 383           $customHeaders
 384         );
 385         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 386         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 387       }
 388       if ($this->_duplicateCount) {
 389         $headers = array_merge(array(ts('Line Number'),
 390             ts('View Contribution URL'),
 391           ),
 392           $customHeaders
 393         );
 394
 395         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 396         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 397       }
 398     }
 399     return $this->fini();
 400   }
 401
 402   /**
 403    * Given a list of the importable field keys that the user has selected
 404    * set the active fields array to this list
 405    *
 406    * @param array mapped array of values
 407    *
 408    * @return void
 409    */
 410   public function setActiveFields($fieldKeys) {
 411     $this->_activeFieldCount = count($fieldKeys);
 412     foreach ($fieldKeys as $key) {
 413       if (empty($this->_fields[$key])) {
 414         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 415       }
 416       else {
 417         $this->_activeFields[] = clone($this->_fields[$key]);
 418       }
 419     }
 420   }
 421
 422   /**
 423    * @param array $elements
 424    */
 425   public function setActiveFieldSoftCredit($elements) {
 426     for ($i = 0; $i < count($elements); $i++) {
 427       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 428     }
 429   }
 430
 431   /**
 432    * @param array $elements
 433    */
 434   public function setActiveFieldSoftCreditType($elements) {
 435     for ($i = 0; $i < count($elements); $i++) {
 436       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 437     }
 438   }
 439
 440   /**
 441    * Format the field values for input to the api
 442    *
 443    * @return array (reference ) associative array of name/value pairs
 444    */
 445   public function &getActiveFieldParams() {
 446     $params = array();
 447     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 448       if (isset($this->_activeFields[$i]->_value)) {
 449         if (isset($this->_activeFields[$i]->_softCreditField)) {
 450           if (!isset($params[$this->_activeFields[$i]->_name])) {
 451             $params[$this->_activeFields[$i]->_name] = array();
 452           }
 453           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 454           if (isset($this->_activeFields[$i]->_softCreditType)) {
 455             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 456           }
 457         }
 458
 459         if (!isset($params[$this->_activeFields[$i]->_name])) {
 460           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 461             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 462           }
 463         }
 464       }
 465     }
 466     return $params;
 467   }
 468
 469   /**
 470    * @param string $name
 471    * @param $title
 472    * @param int $type
 473    * @param string $headerPattern
 474    * @param string $dataPattern
 475    */
 476   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 477     if (empty($name)) {
 478       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 479     }
 480     else {
 481       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 482       if (!array_key_exists($name, $tempField)) {
 483         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 484       }
 485       else {
 486         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 487           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 488         );
 489       }
 490     }
 491   }
 492
 493   /**
 494    * Store parser values
 495    *
 496    * @param CRM_Core_Session $store
 497    *
 498    * @param int $mode
 499    *
 500    * @return void
 501    */
 502   public function set($store, $mode = self::MODE_SUMMARY) {
 503     $store->set('fileSize', $this->_fileSize);
 504     $store->set('lineCount', $this->_lineCount);
 505     $store->set('seperator', $this->_seperator);
 506     $store->set('fields', $this->getSelectValues());
 507     $store->set('fieldTypes', $this->getSelectTypes());
 508
 509     $store->set('headerPatterns', $this->getHeaderPatterns());
 510     $store->set('dataPatterns', $this->getDataPatterns());
 511     $store->set('columnCount', $this->_activeFieldCount);
 512
 513     $store->set('totalRowCount', $this->_totalCount);
 514     $store->set('validRowCount', $this->_validCount);
 515     $store->set('invalidRowCount', $this->_invalidRowCount);
 516     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 517     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 518     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 519     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 520     $store->set('conflictRowCount', $this->_conflictCount);
 521
 522     switch ($this->_contactType) {
 523       case 'Individual':
 524         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 525         break;
 526
 527       case 'Household':
 528         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 529         break;
 530
 531       case 'Organization':
 532         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 533     }
 534
 535     if ($this->_invalidRowCount) {
 536       $store->set('errorsFileName', $this->_errorFileName);
 537     }
 538     if ($this->_conflictCount) {
 539       $store->set('conflictsFileName', $this->_conflictFileName);
 540     }
 541     if (isset($this->_rows) && !empty($this->_rows)) {
 542       $store->set('dataValues', $this->_rows);
 543     }
 544
 545     if ($this->_invalidPledgePaymentRowCount) {
 546       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 547     }
 548
 549     if ($this->_invalidSoftCreditRowCount) {
 550       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 551     }
 552
 553     if ($mode == self::MODE_IMPORT) {
 554       $store->set('duplicateRowCount', $this->_duplicateCount);
 555       if ($this->_duplicateCount) {
 556         $store->set('duplicatesFileName', $this->_duplicateFileName);
 557       }
 558     }
 559   }
 560
 561   /**
 562    * Export data to a CSV file
 563    *
 564    * @param string $fileName
 565    * @param array $header
 566    * @param array $data
 567    *
 568    * @return void
 569    */
 570   public static function exportCSV($fileName, $header, $data) {
 571     $output = array();
 572     $fd = fopen($fileName, 'w');
 573
 574     foreach ($header as $key => $value) {
 575       $header[$key] = "\"$value\"";
 576     }
 577     $config = CRM_Core_Config::singleton();
 578     $output[] = implode($config->fieldSeparator, $header);
 579
 580     foreach ($data as $datum) {
 581       foreach ($datum as $key => $value) {
 582         if (isset($value[0]) && is_array($value)) {
 583           foreach ($value[0] as $k1 => $v1) {
 584             if ($k1 == 'location_type_id') {
 585               continue;
 586             }
 587             $datum[$k1] = $v1;
 588           }
 589         }
 590         else {
 591           $datum[$key] = "\"$value\"";
 592         }
 593       }
 594       $output[] = implode($config->fieldSeparator, $datum);
 595     }
 596     fwrite($fd, implode("\n", $output));
 597     fclose($fd);
 598   }
 599
 600   /**
 601    * Determines the file extension based on error code
 602    *
 603    * @param int $type
 604    *   Error code constant.
 605    *
 606    * @return string
 607    */
 608   public static function errorFileName($type) {
 609     $fileName = NULL;
 610     if (empty($type)) {
 611       return $fileName;
 612     }
 613
 614     $config = CRM_Core_Config::singleton();
 615     $fileName = $config->uploadDir . "sqlImport";
 616
 617     switch ($type) {
 618       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 619         $fileName .= '.softCreditErrors';
 620         break;
 621
 622       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 623         $fileName .= '.pledgePaymentErrors';
 624         break;
 625
 626       default:
 627         $fileName = parent::errorFileName($type);
 628         break;
 629     }
 630
 631     return $fileName;
 632   }
 633
 634   /**
 635    * Determines the file name based on error code
 636    *
 637    * @param int $type
 638    *   Error code constant.
 639    *
 640    * @return string
 641    */
 642   public static function saveFileName($type) {
 643     $fileName = NULL;
 644     if (empty($type)) {
 645       return $fileName;
 646     }
 647
 648     switch ($type) {
 649       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 650         $fileName = 'Import_Soft_Credit_Errors.csv';
 651         break;
 652
 653       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 654         $fileName = 'Import_Pledge_Payment_Errors.csv';
 655         break;
 656
 657       default:
 658         $fileName = parent::saveFileName($type);
 659         break;
 660     }
 661
 662     return $fileName;
 663   }
 664 }