CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.6                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  37
  38   /**
  39    * Contribution-specific result codes
  40    * @see CRM_Import_Parser result code constants
  41    */
  42   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  43
  44   protected $_fileName;
  45
  46   /**
  47    * Imported file size
  48    */
  49   protected $_fileSize;
  50
  51   /**
  52    * Seperator being used
  53    */
  54   protected $_seperator;
  55
  56   /**
  57    * Total number of lines in file
  58    */
  59   protected $_lineCount;
  60
  61   /**
  62    * Running total number of valid soft credit rows
  63    */
  64   protected $_validSoftCreditRowCount;
  65
  66   /**
  67    * Running total number of invalid soft credit rows
  68    */
  69   protected $_invalidSoftCreditRowCount;
  70
  71   /**
  72    * Running total number of valid pledge payment rows
  73    */
  74   protected $_validPledgePaymentRowCount;
  75
  76   /**
  77    * Running total number of invalid pledge payment rows
  78    */
  79   protected $_invalidPledgePaymentRowCount;
  80
  81   /**
  82    * Array of pledge payment error lines, bounded by MAX_ERROR
  83    */
  84   protected $_pledgePaymentErrors;
  85
  86   /**
  87    * Array of pledge payment error lines, bounded by MAX_ERROR
  88    */
  89   protected $_softCreditErrors;
  90
  91   /**
  92    * Filename of pledge payment error data
  93    *
  94    * @var string
  95    */
  96   protected $_pledgePaymentErrorsFileName;
  97
  98   /**
  99    * Filename of soft credit error data
 100    *
 101    * @var string
 102    */
 103   protected $_softCreditErrorsFileName;
 104
 105   /**
 106    * Whether the file has a column header or not
 107    *
 108    * @var boolean
 109    */
 110   protected $_haveColumnHeader;
 111
 112   /**
 113    * @param string $fileName
 114    * @param string $seperator
 115    * @param $mapper
 116    * @param bool $skipColumnHeader
 117    * @param int $mode
 118    * @param int $contactType
 119    * @param int $onDuplicate
 120    *
 121    * @return mixed
 122    * @throws Exception
 123    */
 124   function run($fileName,
 125     $seperator = ',',
 126     &$mapper,
 127     $skipColumnHeader = FALSE,
 128     $mode = self::MODE_PREVIEW,
 129     $contactType = self::CONTACT_INDIVIDUAL,
 130     $onDuplicate = self::DUPLICATE_SKIP
 131   ) {
 132     if (!is_array($fileName)) {
 133       CRM_Core_Error::fatal();
 134     }
 135     $fileName = $fileName['name'];
 136
 137     switch ($contactType) {
 138       case self::CONTACT_INDIVIDUAL:
 139         $this->_contactType = 'Individual';
 140         break;
 141
 142       case self::CONTACT_HOUSEHOLD:
 143         $this->_contactType = 'Household';
 144         break;
 145
 146       case self::CONTACT_ORGANIZATION:
 147         $this->_contactType = 'Organization';
 148     }
 149
 150     $this->init();
 151
 152     $this->_haveColumnHeader = $skipColumnHeader;
 153
 154     $this->_seperator = $seperator;
 155
 156     $fd = fopen($fileName, "r");
 157     if (!$fd) {
 158       return FALSE;
 159     }
 160
 161     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 162     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 163     $this->_totalCount = $this->_conflictCount = 0;
 164
 165     $this->_errors = array();
 166     $this->_warnings = array();
 167     $this->_conflicts = array();
 168     $this->_pledgePaymentErrors = array();
 169     $this->_softCreditErrors = array();
 170
 171     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 172
 173     if ($mode == self::MODE_MAPFIELD) {
 174       $this->_rows = array();
 175     }
 176     else {
 177       $this->_activeFieldCount = count($this->_activeFields);
 178     }
 179
 180     while (!feof($fd)) {
 181       $this->_lineCount++;
 182
 183       $values = fgetcsv($fd, 8192, $seperator);
 184       if (!$values) {
 185         continue;
 186       }
 187
 188       self::encloseScrub($values);
 189
 190       // skip column header if we're not in mapfield mode
 191       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 192         $skipColumnHeader = FALSE;
 193         continue;
 194       }
 195
 196       /* trim whitespace around the values */
 197
 198       $empty = TRUE;
 199       foreach ($values as $k => $v) {
 200         $values[$k] = trim($v, " \t\r\n");
 201       }
 202
 203       if (CRM_Utils_System::isNull($values)) {
 204         continue;
 205       }
 206
 207       $this->_totalCount++;
 208
 209       if ($mode == self::MODE_MAPFIELD) {
 210         $returnCode = $this->mapField($values);
 211       }
 212       elseif ($mode == self::MODE_PREVIEW) {
 213         $returnCode = $this->preview($values);
 214       }
 215       elseif ($mode == self::MODE_SUMMARY) {
 216         $returnCode = $this->summary($values);
 217       }
 218       elseif ($mode == self::MODE_IMPORT) {
 219         $returnCode = $this->import($onDuplicate, $values);
 220       }
 221       else {
 222         $returnCode = self::ERROR;
 223       }
 224
 225       // note that a line could be valid but still produce a warning
 226       if ($returnCode == self::VALID) {
 227         $this->_validCount++;
 228         if ($mode == self::MODE_MAPFIELD) {
 229           $this->_rows[] = $values;
 230           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 231         }
 232       }
 233
 234       if ($returnCode == self::SOFT_CREDIT) {
 235         $this->_validSoftCreditRowCount++;
 236         $this->_validCount++;
 237         if ($mode == self::MODE_MAPFIELD) {
 238           $this->_rows[] = $values;
 239           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 240         }
 241       }
 242
 243       if ($returnCode == self::PLEDGE_PAYMENT) {
 244         $this->_validPledgePaymentRowCount++;
 245         $this->_validCount++;
 246         if ($mode == self::MODE_MAPFIELD) {
 247           $this->_rows[] = $values;
 248           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 249         }
 250       }
 251
 252       if ($returnCode == self::WARNING) {
 253         $this->_warningCount++;
 254         if ($this->_warningCount < $this->_maxWarningCount) {
 255           $this->_warningCount[] = $line;
 256         }
 257       }
 258
 259       if ($returnCode == self::ERROR) {
 260         $this->_invalidRowCount++;
 261         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 262           $recordNumber = $this->_lineCount;
 263           if ($this->_haveColumnHeader) {
 264             $recordNumber--;
 265           }
 266           array_unshift($values, $recordNumber);
 267           $this->_errors[] = $values;
 268         }
 269       }
 270
 271       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 272         $this->_invalidPledgePaymentRowCount++;
 273         if ($this->_invalidPledgePaymentRowCount < $this->_maxErrorCount) {
 274           $recordNumber = $this->_lineCount;
 275           if ($this->_haveColumnHeader) {
 276             $recordNumber--;
 277           }
 278           array_unshift($values, $recordNumber);
 279           $this->_pledgePaymentErrors[] = $values;
 280         }
 281       }
 282
 283       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 284         $this->_invalidSoftCreditRowCount++;
 285         if ($this->_invalidSoftCreditRowCount < $this->_maxErrorCount) {
 286           $recordNumber = $this->_lineCount;
 287           if ($this->_haveColumnHeader) {
 288             $recordNumber--;
 289           }
 290           array_unshift($values, $recordNumber);
 291           $this->_softCreditErrors[] = $values;
 292         }
 293       }
 294
 295       if ($returnCode == self::CONFLICT) {
 296         $this->_conflictCount++;
 297         $recordNumber = $this->_lineCount;
 298         if ($this->_haveColumnHeader) {
 299           $recordNumber--;
 300         }
 301         array_unshift($values, $recordNumber);
 302         $this->_conflicts[] = $values;
 303       }
 304
 305       if ($returnCode == self::DUPLICATE) {
 306         if ($returnCode == self::MULTIPLE_DUPE) {
 307           /* TODO: multi-dupes should be counted apart from singles
 308                      * on non-skip action */
 309         }
 310         $this->_duplicateCount++;
 311         $recordNumber = $this->_lineCount;
 312         if ($this->_haveColumnHeader) {
 313           $recordNumber--;
 314         }
 315         array_unshift($values, $recordNumber);
 316         $this->_duplicates[] = $values;
 317         if ($onDuplicate != self::DUPLICATE_SKIP) {
 318           $this->_validCount++;
 319         }
 320       }
 321
 322       // we give the derived class a way of aborting the process
 323       // note that the return code could be multiple code or'ed together
 324       if ($returnCode == self::STOP) {
 325         break;
 326       }
 327
 328       // if we are done processing the maxNumber of lines, break
 329       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 330         break;
 331       }
 332     }
 333
 334     fclose($fd);
 335
 336     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 337       $customHeaders = $mapper;
 338
 339       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 340       foreach ($customHeaders as $key => $value) {
 341         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 342           $customHeaders[$key] = $customfields[$id][0];
 343         }
 344       }
 345       if ($this->_invalidRowCount) {
 346         // removed view url for invlaid contacts
 347         $headers = array_merge(array(ts('Line Number'),
 348             ts('Reason'),
 349           ),
 350           $customHeaders
 351         );
 352         $this->_errorFileName = self::errorFileName(self::ERROR);
 353         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 354       }
 355
 356       if ($this->_invalidPledgePaymentRowCount) {
 357         // removed view url for invlaid contacts
 358         $headers = array_merge(array(ts('Line Number'),
 359             ts('Reason'),
 360           ),
 361           $customHeaders
 362         );
 363         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 364         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 365       }
 366
 367       if ($this->_invalidSoftCreditRowCount) {
 368         // removed view url for invlaid contacts
 369         $headers = array_merge(array(ts('Line Number'),
 370             ts('Reason'),
 371           ),
 372           $customHeaders
 373         );
 374         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 375         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 376       }
 377
 378       if ($this->_conflictCount) {
 379         $headers = array_merge(array(ts('Line Number'),
 380             ts('Reason'),
 381           ),
 382           $customHeaders
 383         );
 384         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 385         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 386       }
 387       if ($this->_duplicateCount) {
 388         $headers = array_merge(array(ts('Line Number'),
 389             ts('View Contribution URL'),
 390           ),
 391           $customHeaders
 392         );
 393
 394         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 395         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 396       }
 397     }
 398     return $this->fini();
 399   }
 400
 401   /**
 402    * Given a list of the importable field keys that the user has selected
 403    * set the active fields array to this list
 404    *
 405    * @param array mapped array of values
 406    *
 407    * @return void
 408    * @access public
 409    */
 410   function setActiveFields($fieldKeys) {
 411     $this->_activeFieldCount = count($fieldKeys);
 412     foreach ($fieldKeys as $key) {
 413       if (empty($this->_fields[$key])) {
 414         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 415       }
 416       else {
 417         $this->_activeFields[] = clone($this->_fields[$key]);
 418       }
 419     }
 420   }
 421
 422   /**
 423    * @param array $elements
 424    */
 425   function setActiveFieldSoftCredit($elements) {
 426     for ($i = 0; $i < count($elements); $i++) {
 427       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 428     }
 429   }
 430
 431   /**
 432    * @param array $elements
 433    */
 434   function setActiveFieldSoftCreditType($elements) {
 435     for ($i = 0; $i < count($elements); $i++) {
 436       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 437     }
 438   }
 439
 440   /**
 441    * Format the field values for input to the api
 442    *
 443    * @return array (reference ) associative array of name/value pairs
 444    * @access public
 445    */
 446   function &getActiveFieldParams() {
 447     $params = array();
 448     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 449       if (isset($this->_activeFields[$i]->_value)) {
 450         if (isset($this->_activeFields[$i]->_softCreditField)) {
 451           if (!isset($params[$this->_activeFields[$i]->_name])) {
 452             $params[$this->_activeFields[$i]->_name] = array();
 453           }
 454           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 455           if(isset($this->_activeFields[$i]->_softCreditType)){
 456             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 457           }
 458         }
 459
 460         if (!isset($params[$this->_activeFields[$i]->_name])) {
 461           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 462             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 463           }
 464         }
 465       }
 466     }
 467     return $params;
 468   }
 469
 470   /**
 471    * @param string $name
 472    * @param $title
 473    * @param int $type
 474    * @param string $headerPattern
 475    * @param string $dataPattern
 476    */
 477   function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 478     if (empty($name)) {
 479       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 480     }
 481     else {
 482       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 483       if (!array_key_exists($name, $tempField)) {
 484         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 485       }
 486       else {
 487         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 488           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 489         );
 490       }
 491     }
 492   }
 493
 494   /**
 495    * Store parser values
 496    *
 497    * @param CRM_Core_Session $store
 498    *
 499    * @param int $mode
 500    *
 501    * @return void
 502    * @access public
 503    */
 504   function set($store, $mode = self::MODE_SUMMARY) {
 505     $store->set('fileSize', $this->_fileSize);
 506     $store->set('lineCount', $this->_lineCount);
 507     $store->set('seperator', $this->_seperator);
 508     $store->set('fields', $this->getSelectValues());
 509     $store->set('fieldTypes', $this->getSelectTypes());
 510
 511     $store->set('headerPatterns', $this->getHeaderPatterns());
 512     $store->set('dataPatterns', $this->getDataPatterns());
 513     $store->set('columnCount', $this->_activeFieldCount);
 514
 515     $store->set('totalRowCount', $this->_totalCount);
 516     $store->set('validRowCount', $this->_validCount);
 517     $store->set('invalidRowCount', $this->_invalidRowCount);
 518     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 519     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 520     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 521     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 522     $store->set('conflictRowCount', $this->_conflictCount);
 523
 524     switch ($this->_contactType) {
 525       case 'Individual':
 526         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 527         break;
 528
 529       case 'Household':
 530         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 531         break;
 532
 533       case 'Organization':
 534         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 535     }
 536
 537     if ($this->_invalidRowCount) {
 538       $store->set('errorsFileName', $this->_errorFileName);
 539     }
 540     if ($this->_conflictCount) {
 541       $store->set('conflictsFileName', $this->_conflictFileName);
 542     }
 543     if (isset($this->_rows) && !empty($this->_rows)) {
 544       $store->set('dataValues', $this->_rows);
 545     }
 546
 547     if ($this->_invalidPledgePaymentRowCount) {
 548       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 549     }
 550
 551     if ($this->_invalidSoftCreditRowCount) {
 552       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 553     }
 554
 555     if ($mode == self::MODE_IMPORT) {
 556       $store->set('duplicateRowCount', $this->_duplicateCount);
 557       if ($this->_duplicateCount) {
 558         $store->set('duplicatesFileName', $this->_duplicateFileName);
 559       }
 560     }
 561   }
 562
 563   /**
 564    * Export data to a CSV file
 565    *
 566    * @param string $fileName
 567    * @param array $header
 568    * @param array $data
 569    *
 570    * @return void
 571    * @access public
 572    */
 573   static function exportCSV($fileName, $header, $data) {
 574     $output = array();
 575     $fd = fopen($fileName, 'w');
 576
 577     foreach ($header as $key => $value) {
 578       $header[$key] = "\"$value\"";
 579     }
 580     $config = CRM_Core_Config::singleton();
 581     $output[] = implode($config->fieldSeparator, $header);
 582
 583     foreach ($data as $datum) {
 584       foreach ($datum as $key => $value) {
 585         if (isset($value[0]) && is_array($value)) {
 586           foreach ($value[0] as $k1 => $v1) {
 587             if ($k1 == 'location_type_id') {
 588               continue;
 589             }
 590             $datum[$k1] = $v1;
 591           }
 592         }
 593         else {
 594           $datum[$key] = "\"$value\"";
 595         }
 596       }
 597       $output[] = implode($config->fieldSeparator, $datum);
 598     }
 599     fwrite($fd, implode("\n", $output));
 600     fclose($fd);
 601   }
 602
 603   /**
 604    * Determines the file extension based on error code
 605    *
 606    * @param int $type error code constant
 607    *
 608    * @return string
 609    */
 610   static function errorFileName($type) {
 611     $fileName = NULL;
 612     if (empty($type)) {
 613       return $fileName;
 614     }
 615
 616     $config = CRM_Core_Config::singleton();
 617     $fileName = $config->uploadDir . "sqlImport";
 618
 619     switch ($type) {
 620       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 621         $fileName .= '.softCreditErrors';
 622         break;
 623
 624       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 625         $fileName .= '.pledgePaymentErrors';
 626         break;
 627
 628       default:
 629         $fileName = parent::errorFileName($type);
 630         break;
 631     }
 632
 633     return $fileName;
 634   }
 635
 636   /**
 637    * Determines the file name based on error code
 638    *
 639    * @param int $type error code constant
 640    *
 641    * @return string
 642    */
 643   static function saveFileName($type) {
 644     $fileName = NULL;
 645     if (empty($type)) {
 646       return $fileName;
 647     }
 648
 649     switch ($type) {
 650       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 651         $fileName = 'Import_Soft_Credit_Errors.csv';
 652         break;
 653
 654       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 655         $fileName = 'Import_Pledge_Payment_Errors.csv';
 656         break;
 657
 658       default:
 659         $fileName = parent::saveFileName($type);
 660         break;
 661     }
 662
 663     return $fileName;
 664   }
 665 }
 666