CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.5                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36
  37 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  38
  39   /**
  40    * Contribution-specific result codes
  41    * @see CRM_Import_Parser result code constants
  42    */
  43   CONST SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  44
  45   protected $_fileName;
  46
  47   /**#@+
  48    * @access protected
  49    * @var integer
  50    */
  51
  52   /**
  53    * imported file size
  54    */
  55   protected $_fileSize;
  56
  57   /**
  58    * seperator being used
  59    */
  60   protected $_seperator;
  61
  62   /**
  63    * total number of lines in file
  64    */
  65   protected $_lineCount;
  66
  67   /**
  68    * running total number of valid soft credit rows
  69    */
  70   protected $_validSoftCreditRowCount;
  71
  72   /**
  73    * running total number of invalid soft credit rows
  74    */
  75   protected $_invalidSoftCreditRowCount;
  76
  77   /**
  78    * running total number of valid pledge payment rows
  79    */
  80   protected $_validPledgePaymentRowCount;
  81
  82   /**
  83    * running total number of invalid pledge payment rows
  84    */
  85   protected $_invalidPledgePaymentRowCount;
  86
  87   /**
  88    * array of pledge payment error lines, bounded by MAX_ERROR
  89    */
  90   protected $_pledgePaymentErrors;
  91
  92   /**
  93    * array of pledge payment error lines, bounded by MAX_ERROR
  94    */
  95   protected $_softCreditErrors;
  96
  97   /**
  98    * filename of pledge payment error data
  99    *
 100    * @var string
 101    */
 102   protected $_pledgePaymentErrorsFileName;
 103
 104   /**
 105    * filename of soft credit error data
 106    *
 107    * @var string
 108    */
 109   protected $_softCreditErrorsFileName;
 110
 111   /**
 112    * whether the file has a column header or not
 113    *
 114    * @var boolean
 115    */
 116   protected $_haveColumnHeader;
 117
 118   function run($fileName,
 119     $seperator = ',',
 120     &$mapper,
 121     $skipColumnHeader = FALSE,
 122     $mode = self::MODE_PREVIEW,
 123     $contactType = self::CONTACT_INDIVIDUAL,
 124     $onDuplicate = self::DUPLICATE_SKIP
 125   ) {
 126     if (!is_array($fileName)) {
 127       CRM_Core_Error::fatal();
 128     }
 129     $fileName = $fileName['name'];
 130
 131     switch ($contactType) {
 132       case self::CONTACT_INDIVIDUAL:
 133         $this->_contactType = 'Individual';
 134         break;
 135
 136       case self::CONTACT_HOUSEHOLD:
 137         $this->_contactType = 'Household';
 138         break;
 139
 140       case self::CONTACT_ORGANIZATION:
 141         $this->_contactType = 'Organization';
 142     }
 143
 144     $this->init();
 145
 146     $this->_haveColumnHeader = $skipColumnHeader;
 147
 148     $this->_seperator = $seperator;
 149
 150     $fd = fopen($fileName, "r");
 151     if (!$fd) {
 152       return FALSE;
 153     }
 154
 155     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 156     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 157     $this->_totalCount = $this->_conflictCount = 0;
 158
 159     $this->_errors = array();
 160     $this->_warnings = array();
 161     $this->_conflicts = array();
 162     $this->_pledgePaymentErrors = array();
 163     $this->_softCreditErrors = array();
 164
 165     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 166
 167     if ($mode == self::MODE_MAPFIELD) {
 168       $this->_rows = array();
 169     }
 170     else {
 171       $this->_activeFieldCount = count($this->_activeFields);
 172     }
 173
 174     while (!feof($fd)) {
 175       $this->_lineCount++;
 176
 177       $values = fgetcsv($fd, 8192, $seperator);
 178       if (!$values) {
 179         continue;
 180       }
 181
 182       self::encloseScrub($values);
 183
 184       // skip column header if we're not in mapfield mode
 185       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 186         $skipColumnHeader = FALSE;
 187         continue;
 188       }
 189
 190       /* trim whitespace around the values */
 191
 192       $empty = TRUE;
 193       foreach ($values as $k => $v) {
 194         $values[$k] = trim($v, " \t\r\n");
 195       }
 196
 197       if (CRM_Utils_System::isNull($values)) {
 198         continue;
 199       }
 200
 201       $this->_totalCount++;
 202
 203       if ($mode == self::MODE_MAPFIELD) {
 204         $returnCode = $this->mapField($values);
 205       }
 206       elseif ($mode == self::MODE_PREVIEW) {
 207         $returnCode = $this->preview($values);
 208       }
 209       elseif ($mode == self::MODE_SUMMARY) {
 210         $returnCode = $this->summary($values);
 211       }
 212       elseif ($mode == self::MODE_IMPORT) {
 213         $returnCode = $this->import($onDuplicate, $values);
 214       }
 215       else {
 216         $returnCode = self::ERROR;
 217       }
 218
 219       // note that a line could be valid but still produce a warning
 220       if ($returnCode == self::VALID) {
 221         $this->_validCount++;
 222         if ($mode == self::MODE_MAPFIELD) {
 223           $this->_rows[] = $values;
 224           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 225         }
 226       }
 227
 228       if ($returnCode == self::SOFT_CREDIT) {
 229         $this->_validSoftCreditRowCount++;
 230         $this->_validCount++;
 231         if ($mode == self::MODE_MAPFIELD) {
 232           $this->_rows[] = $values;
 233           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 234         }
 235       }
 236
 237       if ($returnCode == self::PLEDGE_PAYMENT) {
 238         $this->_validPledgePaymentRowCount++;
 239         $this->_validCount++;
 240         if ($mode == self::MODE_MAPFIELD) {
 241           $this->_rows[] = $values;
 242           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 243         }
 244       }
 245
 246       if ($returnCode == self::WARNING) {
 247         $this->_warningCount++;
 248         if ($this->_warningCount < $this->_maxWarningCount) {
 249           $this->_warningCount[] = $line;
 250         }
 251       }
 252
 253       if ($returnCode == self::ERROR) {
 254         $this->_invalidRowCount++;
 255         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 256           $recordNumber = $this->_lineCount;
 257           if ($this->_haveColumnHeader) {
 258             $recordNumber--;
 259           }
 260           array_unshift($values, $recordNumber);
 261           $this->_errors[] = $values;
 262         }
 263       }
 264
 265       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 266         $this->_invalidPledgePaymentRowCount++;
 267         if ($this->_invalidPledgePaymentRowCount < $this->_maxErrorCount) {
 268           $recordNumber = $this->_lineCount;
 269           if ($this->_haveColumnHeader) {
 270             $recordNumber--;
 271           }
 272           array_unshift($values, $recordNumber);
 273           $this->_pledgePaymentErrors[] = $values;
 274         }
 275       }
 276
 277       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 278         $this->_invalidSoftCreditRowCount++;
 279         if ($this->_invalidSoftCreditRowCount < $this->_maxErrorCount) {
 280           $recordNumber = $this->_lineCount;
 281           if ($this->_haveColumnHeader) {
 282             $recordNumber--;
 283           }
 284           array_unshift($values, $recordNumber);
 285           $this->_softCreditErrors[] = $values;
 286         }
 287       }
 288
 289       if ($returnCode == self::CONFLICT) {
 290         $this->_conflictCount++;
 291         $recordNumber = $this->_lineCount;
 292         if ($this->_haveColumnHeader) {
 293           $recordNumber--;
 294         }
 295         array_unshift($values, $recordNumber);
 296         $this->_conflicts[] = $values;
 297       }
 298
 299       if ($returnCode == self::DUPLICATE) {
 300         if ($returnCode == self::MULTIPLE_DUPE) {
 301           /* TODO: multi-dupes should be counted apart from singles
 302                      * on non-skip action */
 303         }
 304         $this->_duplicateCount++;
 305         $recordNumber = $this->_lineCount;
 306         if ($this->_haveColumnHeader) {
 307           $recordNumber--;
 308         }
 309         array_unshift($values, $recordNumber);
 310         $this->_duplicates[] = $values;
 311         if ($onDuplicate != self::DUPLICATE_SKIP) {
 312           $this->_validCount++;
 313         }
 314       }
 315
 316       // we give the derived class a way of aborting the process
 317       // note that the return code could be multiple code or'ed together
 318       if ($returnCode == self::STOP) {
 319         break;
 320       }
 321
 322       // if we are done processing the maxNumber of lines, break
 323       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 324         break;
 325       }
 326     }
 327
 328     fclose($fd);
 329
 330     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 331       $customHeaders = $mapper;
 332
 333       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 334       foreach ($customHeaders as $key => $value) {
 335         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 336           $customHeaders[$key] = $customfields[$id][0];
 337         }
 338       }
 339       if ($this->_invalidRowCount) {
 340         // removed view url for invlaid contacts
 341         $headers = array_merge(array(ts('Line Number'),
 342             ts('Reason'),
 343           ),
 344           $customHeaders
 345         );
 346         $this->_errorFileName = self::errorFileName(self::ERROR);
 347         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 348       }
 349
 350       if ($this->_invalidPledgePaymentRowCount) {
 351         // removed view url for invlaid contacts
 352         $headers = array_merge(array(ts('Line Number'),
 353             ts('Reason'),
 354           ),
 355           $customHeaders
 356         );
 357         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 358         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 359       }
 360
 361       if ($this->_invalidSoftCreditRowCount) {
 362         // removed view url for invlaid contacts
 363         $headers = array_merge(array(ts('Line Number'),
 364             ts('Reason'),
 365           ),
 366           $customHeaders
 367         );
 368         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 369         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 370       }
 371
 372       if ($this->_conflictCount) {
 373         $headers = array_merge(array(ts('Line Number'),
 374             ts('Reason'),
 375           ),
 376           $customHeaders
 377         );
 378         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 379         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 380       }
 381       if ($this->_duplicateCount) {
 382         $headers = array_merge(array(ts('Line Number'),
 383             ts('View Contribution URL'),
 384           ),
 385           $customHeaders
 386         );
 387
 388         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 389         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 390       }
 391     }
 392     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 393     return $this->fini();
 394   }
 395
 396   /**
 397    * Given a list of the importable field keys that the user has selected
 398    * set the active fields array to this list
 399    *
 400    * @param array mapped array of values
 401    *
 402 pppp   * @return void
 403    * @access public
 404    */
 405   function setActiveFields($fieldKeys) {
 406     $this->_activeFieldCount = count($fieldKeys);
 407     foreach ($fieldKeys as $key) {
 408       if (empty($this->_fields[$key])) {
 409         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 410       }
 411       else {
 412         $this->_activeFields[] = clone($this->_fields[$key]);
 413       }
 414     }
 415   }
 416
 417   function setActiveFieldSoftCredit($elements) {
 418     for ($i = 0; $i < count($elements); $i++) {
 419       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 420     }
 421   }
 422
 423   function setActiveFieldSoftCreditType($elements) {
 424     for ($i = 0; $i < count($elements); $i++) {
 425       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 426     }
 427   }
 428   /**
 429    * function to format the field values for input to the api
 430    *
 431    * @return array (reference ) associative array of name/value pairs
 432    * @access public
 433    */
 434   function &getActiveFieldParams() {
 435     $params = array();
 436     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 437       if (isset($this->_activeFields[$i]->_value)) {
 438         if (isset($this->_activeFields[$i]->_softCreditField)) {
 439           if (!isset($params[$this->_activeFields[$i]->_name])) {
 440             $params[$this->_activeFields[$i]->_name] = array();
 441           }
 442           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 443           if(isset($this->_activeFields[$i]->_softCreditType)){
 444             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 445           }
 446         }
 447
 448         if (!isset($params[$this->_activeFields[$i]->_name])) {
 449           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 450             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 451           }
 452         }
 453       }
 454     }
 455     return $params;
 456   }
 457
 458   function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 459     if (empty($name)) {
 460       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 461     }
 462     else {
 463       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 464       if (!array_key_exists($name, $tempField)) {
 465         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 466       }
 467       else {
 468         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 469           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 470         );
 471       }
 472     }
 473   }
 474
 475   /**
 476    * Store parser values
 477    *
 478    * @param CRM_Core_Session $store
 479    *
 480    * @return void
 481    * @access public
 482    */
 483   function set($store, $mode = self::MODE_SUMMARY) {
 484     $store->set('fileSize', $this->_fileSize);
 485     $store->set('lineCount', $this->_lineCount);
 486     $store->set('seperator', $this->_seperator);
 487     $store->set('fields', $this->getSelectValues());
 488     $store->set('fieldTypes', $this->getSelectTypes());
 489
 490     $store->set('headerPatterns', $this->getHeaderPatterns());
 491     $store->set('dataPatterns', $this->getDataPatterns());
 492     $store->set('columnCount', $this->_activeFieldCount);
 493
 494     $store->set('totalRowCount', $this->_totalCount);
 495     $store->set('validRowCount', $this->_validCount);
 496     $store->set('invalidRowCount', $this->_invalidRowCount);
 497     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 498     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 499     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 500     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 501     $store->set('conflictRowCount', $this->_conflictCount);
 502
 503     switch ($this->_contactType) {
 504       case 'Individual':
 505         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 506         break;
 507
 508       case 'Household':
 509         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 510         break;
 511
 512       case 'Organization':
 513         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 514     }
 515
 516     if ($this->_invalidRowCount) {
 517       $store->set('errorsFileName', $this->_errorFileName);
 518     }
 519     if ($this->_conflictCount) {
 520       $store->set('conflictsFileName', $this->_conflictFileName);
 521     }
 522     if (isset($this->_rows) && !empty($this->_rows)) {
 523       $store->set('dataValues', $this->_rows);
 524     }
 525
 526     if ($this->_invalidPledgePaymentRowCount) {
 527       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 528     }
 529
 530     if ($this->_invalidSoftCreditRowCount) {
 531       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 532     }
 533
 534     if ($mode == self::MODE_IMPORT) {
 535       $store->set('duplicateRowCount', $this->_duplicateCount);
 536       if ($this->_duplicateCount) {
 537         $store->set('duplicatesFileName', $this->_duplicateFileName);
 538       }
 539     }
 540     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 541   }
 542
 543   /**
 544    * Export data to a CSV file
 545    *
 546    * @param string $filename
 547    * @param array $header
 548    * @param data $data
 549    *
 550    * @return void
 551    * @access public
 552    */
 553   static function exportCSV($fileName, $header, $data) {
 554     $output = array();
 555     $fd = fopen($fileName, 'w');
 556
 557     foreach ($header as $key => $value) {
 558       $header[$key] = "\"$value\"";
 559     }
 560     $config = CRM_Core_Config::singleton();
 561     $output[] = implode($config->fieldSeparator, $header);
 562
 563     foreach ($data as $datum) {
 564       foreach ($datum as $key => $value) {
 565         if (isset($value[0]) && is_array($value)) {
 566           foreach ($value[0] as $k1 => $v1) {
 567             if ($k1 == 'location_type_id') {
 568               continue;
 569             }
 570             $datum[$k1] = $v1;
 571           }
 572         }
 573         else {
 574           $datum[$key] = "\"$value\"";
 575         }
 576       }
 577       $output[] = implode($config->fieldSeparator, $datum);
 578     }
 579     fwrite($fd, implode("\n", $output));
 580     fclose($fd);
 581   }
 582
 583   static function errorFileName($type) {
 584     $fileName = NULL;
 585     if (empty($type)) {
 586       return $fileName;
 587     }
 588
 589     $config = CRM_Core_Config::singleton();
 590     $fileName = $config->uploadDir . "sqlImport";
 591
 592     switch ($type) {
 593       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 594         $fileName .= '.softCreditErrors';
 595         break;
 596
 597       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 598         $fileName .= '.pledgePaymentErrors';
 599         break;
 600
 601       default:
 602         $fileName = parent::errorFileName($type);
 603         break;
 604     }
 605
 606     return $fileName;
 607   }
 608
 609   static function saveFileName($type) {
 610     $fileName = NULL;
 611     if (empty($type)) {
 612       return $fileName;
 613     }
 614
 615     switch ($type) {
 616       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 617         $fileName = 'Import_Soft_Credit_Errors.csv';
 618         break;
 619
 620       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 621         $fileName = 'Import_Pledge_Payment_Errors.csv';
 622         break;
 623
 624       default:
 625         $fileName = parent::saveFileName($type);
 626         break;
 627     }
 628
 629     return $fileName;
 630   }
 631 }
 632