CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.5                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2014                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2014
  32  * $Id$
  33  *
  34  */
  35
  36
  37 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  38
  39   /**
  40    * Contribution-specific result codes
  41    * @see CRM_Import_Parser result code constants
  42    */
  43   CONST SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  44
  45   protected $_fileName;
  46
  47   /**#@+
  48    * @access protected
  49    * @var integer
  50    */
  51
  52   /**
  53    * imported file size
  54    */
  55   protected $_fileSize;
  56
  57   /**
  58    * seperator being used
  59    */
  60   protected $_seperator;
  61
  62   /**
  63    * total number of lines in file
  64    */
  65   protected $_lineCount;
  66
  67   /**
  68    * running total number of valid soft credit rows
  69    */
  70   protected $_validSoftCreditRowCount;
  71
  72   /**
  73    * running total number of invalid soft credit rows
  74    */
  75   protected $_invalidSoftCreditRowCount;
  76
  77   /**
  78    * running total number of valid pledge payment rows
  79    */
  80   protected $_validPledgePaymentRowCount;
  81
  82   /**
  83    * running total number of invalid pledge payment rows
  84    */
  85   protected $_invalidPledgePaymentRowCount;
  86
  87   /**
  88    * array of pledge payment error lines, bounded by MAX_ERROR
  89    */
  90   protected $_pledgePaymentErrors;
  91
  92   /**
  93    * array of pledge payment error lines, bounded by MAX_ERROR
  94    */
  95   protected $_softCreditErrors;
  96
  97   /**
  98    * filename of pledge payment error data
  99    *
 100    * @var string
 101    */
 102   protected $_pledgePaymentErrorsFileName;
 103
 104   /**
 105    * filename of soft credit error data
 106    *
 107    * @var string
 108    */
 109   protected $_softCreditErrorsFileName;
 110
 111   /**
 112    * whether the file has a column header or not
 113    *
 114    * @var boolean
 115    */
 116   protected $_haveColumnHeader;
 117
 118   /**
 119    * @param $fileName
 120    * @param string $seperator
 121    * @param $mapper
 122    * @param bool $skipColumnHeader
 123    * @param int $mode
 124    * @param int $contactType
 125    * @param int $onDuplicate
 126    *
 127    * @return mixed
 128    * @throws Exception
 129    */
 130   function run($fileName,
 131     $seperator = ',',
 132     &$mapper,
 133     $skipColumnHeader = FALSE,
 134     $mode = self::MODE_PREVIEW,
 135     $contactType = self::CONTACT_INDIVIDUAL,
 136     $onDuplicate = self::DUPLICATE_SKIP
 137   ) {
 138     if (!is_array($fileName)) {
 139       CRM_Core_Error::fatal();
 140     }
 141     $fileName = $fileName['name'];
 142
 143     switch ($contactType) {
 144       case self::CONTACT_INDIVIDUAL:
 145         $this->_contactType = 'Individual';
 146         break;
 147
 148       case self::CONTACT_HOUSEHOLD:
 149         $this->_contactType = 'Household';
 150         break;
 151
 152       case self::CONTACT_ORGANIZATION:
 153         $this->_contactType = 'Organization';
 154     }
 155
 156     $this->init();
 157
 158     $this->_haveColumnHeader = $skipColumnHeader;
 159
 160     $this->_seperator = $seperator;
 161
 162     $fd = fopen($fileName, "r");
 163     if (!$fd) {
 164       return FALSE;
 165     }
 166
 167     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 168     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 169     $this->_totalCount = $this->_conflictCount = 0;
 170
 171     $this->_errors = array();
 172     $this->_warnings = array();
 173     $this->_conflicts = array();
 174     $this->_pledgePaymentErrors = array();
 175     $this->_softCreditErrors = array();
 176
 177     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 178
 179     if ($mode == self::MODE_MAPFIELD) {
 180       $this->_rows = array();
 181     }
 182     else {
 183       $this->_activeFieldCount = count($this->_activeFields);
 184     }
 185
 186     while (!feof($fd)) {
 187       $this->_lineCount++;
 188
 189       $values = fgetcsv($fd, 8192, $seperator);
 190       if (!$values) {
 191         continue;
 192       }
 193
 194       self::encloseScrub($values);
 195
 196       // skip column header if we're not in mapfield mode
 197       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 198         $skipColumnHeader = FALSE;
 199         continue;
 200       }
 201
 202       /* trim whitespace around the values */
 203
 204       $empty = TRUE;
 205       foreach ($values as $k => $v) {
 206         $values[$k] = trim($v, " \t\r\n");
 207       }
 208
 209       if (CRM_Utils_System::isNull($values)) {
 210         continue;
 211       }
 212
 213       $this->_totalCount++;
 214
 215       if ($mode == self::MODE_MAPFIELD) {
 216         $returnCode = $this->mapField($values);
 217       }
 218       elseif ($mode == self::MODE_PREVIEW) {
 219         $returnCode = $this->preview($values);
 220       }
 221       elseif ($mode == self::MODE_SUMMARY) {
 222         $returnCode = $this->summary($values);
 223       }
 224       elseif ($mode == self::MODE_IMPORT) {
 225         $returnCode = $this->import($onDuplicate, $values);
 226       }
 227       else {
 228         $returnCode = self::ERROR;
 229       }
 230
 231       // note that a line could be valid but still produce a warning
 232       if ($returnCode == self::VALID) {
 233         $this->_validCount++;
 234         if ($mode == self::MODE_MAPFIELD) {
 235           $this->_rows[] = $values;
 236           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 237         }
 238       }
 239
 240       if ($returnCode == self::SOFT_CREDIT) {
 241         $this->_validSoftCreditRowCount++;
 242         $this->_validCount++;
 243         if ($mode == self::MODE_MAPFIELD) {
 244           $this->_rows[] = $values;
 245           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 246         }
 247       }
 248
 249       if ($returnCode == self::PLEDGE_PAYMENT) {
 250         $this->_validPledgePaymentRowCount++;
 251         $this->_validCount++;
 252         if ($mode == self::MODE_MAPFIELD) {
 253           $this->_rows[] = $values;
 254           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 255         }
 256       }
 257
 258       if ($returnCode == self::WARNING) {
 259         $this->_warningCount++;
 260         if ($this->_warningCount < $this->_maxWarningCount) {
 261           $this->_warningCount[] = $line;
 262         }
 263       }
 264
 265       if ($returnCode == self::ERROR) {
 266         $this->_invalidRowCount++;
 267         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 268           $recordNumber = $this->_lineCount;
 269           if ($this->_haveColumnHeader) {
 270             $recordNumber--;
 271           }
 272           array_unshift($values, $recordNumber);
 273           $this->_errors[] = $values;
 274         }
 275       }
 276
 277       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 278         $this->_invalidPledgePaymentRowCount++;
 279         if ($this->_invalidPledgePaymentRowCount < $this->_maxErrorCount) {
 280           $recordNumber = $this->_lineCount;
 281           if ($this->_haveColumnHeader) {
 282             $recordNumber--;
 283           }
 284           array_unshift($values, $recordNumber);
 285           $this->_pledgePaymentErrors[] = $values;
 286         }
 287       }
 288
 289       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 290         $this->_invalidSoftCreditRowCount++;
 291         if ($this->_invalidSoftCreditRowCount < $this->_maxErrorCount) {
 292           $recordNumber = $this->_lineCount;
 293           if ($this->_haveColumnHeader) {
 294             $recordNumber--;
 295           }
 296           array_unshift($values, $recordNumber);
 297           $this->_softCreditErrors[] = $values;
 298         }
 299       }
 300
 301       if ($returnCode == self::CONFLICT) {
 302         $this->_conflictCount++;
 303         $recordNumber = $this->_lineCount;
 304         if ($this->_haveColumnHeader) {
 305           $recordNumber--;
 306         }
 307         array_unshift($values, $recordNumber);
 308         $this->_conflicts[] = $values;
 309       }
 310
 311       if ($returnCode == self::DUPLICATE) {
 312         if ($returnCode == self::MULTIPLE_DUPE) {
 313           /* TODO: multi-dupes should be counted apart from singles
 314                      * on non-skip action */
 315         }
 316         $this->_duplicateCount++;
 317         $recordNumber = $this->_lineCount;
 318         if ($this->_haveColumnHeader) {
 319           $recordNumber--;
 320         }
 321         array_unshift($values, $recordNumber);
 322         $this->_duplicates[] = $values;
 323         if ($onDuplicate != self::DUPLICATE_SKIP) {
 324           $this->_validCount++;
 325         }
 326       }
 327
 328       // we give the derived class a way of aborting the process
 329       // note that the return code could be multiple code or'ed together
 330       if ($returnCode == self::STOP) {
 331         break;
 332       }
 333
 334       // if we are done processing the maxNumber of lines, break
 335       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 336         break;
 337       }
 338     }
 339
 340     fclose($fd);
 341
 342     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 343       $customHeaders = $mapper;
 344
 345       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 346       foreach ($customHeaders as $key => $value) {
 347         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 348           $customHeaders[$key] = $customfields[$id][0];
 349         }
 350       }
 351       if ($this->_invalidRowCount) {
 352         // removed view url for invlaid contacts
 353         $headers = array_merge(array(ts('Line Number'),
 354             ts('Reason'),
 355           ),
 356           $customHeaders
 357         );
 358         $this->_errorFileName = self::errorFileName(self::ERROR);
 359         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 360       }
 361
 362       if ($this->_invalidPledgePaymentRowCount) {
 363         // removed view url for invlaid contacts
 364         $headers = array_merge(array(ts('Line Number'),
 365             ts('Reason'),
 366           ),
 367           $customHeaders
 368         );
 369         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 370         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 371       }
 372
 373       if ($this->_invalidSoftCreditRowCount) {
 374         // removed view url for invlaid contacts
 375         $headers = array_merge(array(ts('Line Number'),
 376             ts('Reason'),
 377           ),
 378           $customHeaders
 379         );
 380         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 381         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 382       }
 383
 384       if ($this->_conflictCount) {
 385         $headers = array_merge(array(ts('Line Number'),
 386             ts('Reason'),
 387           ),
 388           $customHeaders
 389         );
 390         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 391         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 392       }
 393       if ($this->_duplicateCount) {
 394         $headers = array_merge(array(ts('Line Number'),
 395             ts('View Contribution URL'),
 396           ),
 397           $customHeaders
 398         );
 399
 400         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 401         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 402       }
 403     }
 404     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 405     return $this->fini();
 406   }
 407
 408   /**
 409    * Given a list of the importable field keys that the user has selected
 410    * set the active fields array to this list
 411    *
 412    * @param array mapped array of values
 413    *
 414 pppp   * @return void
 415    * @access public
 416    */
 417   function setActiveFields($fieldKeys) {
 418     $this->_activeFieldCount = count($fieldKeys);
 419     foreach ($fieldKeys as $key) {
 420       if (empty($this->_fields[$key])) {
 421         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 422       }
 423       else {
 424         $this->_activeFields[] = clone($this->_fields[$key]);
 425       }
 426     }
 427   }
 428
 429   /**
 430    * @param $elements
 431    */
 432   function setActiveFieldSoftCredit($elements) {
 433     for ($i = 0; $i < count($elements); $i++) {
 434       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 435     }
 436   }
 437
 438   /**
 439    * @param $elements
 440    */
 441   function setActiveFieldSoftCreditType($elements) {
 442     for ($i = 0; $i < count($elements); $i++) {
 443       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 444     }
 445   }
 446   /**
 447    * function to format the field values for input to the api
 448    *
 449    * @return array (reference ) associative array of name/value pairs
 450    * @access public
 451    */
 452   function &getActiveFieldParams() {
 453     $params = array();
 454     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 455       if (isset($this->_activeFields[$i]->_value)) {
 456         if (isset($this->_activeFields[$i]->_softCreditField)) {
 457           if (!isset($params[$this->_activeFields[$i]->_name])) {
 458             $params[$this->_activeFields[$i]->_name] = array();
 459           }
 460           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 461           if(isset($this->_activeFields[$i]->_softCreditType)){
 462             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 463           }
 464         }
 465
 466         if (!isset($params[$this->_activeFields[$i]->_name])) {
 467           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 468             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 469           }
 470         }
 471       }
 472     }
 473     return $params;
 474   }
 475
 476   /**
 477    * @param $name
 478    * @param $title
 479    * @param int $type
 480    * @param string $headerPattern
 481    * @param string $dataPattern
 482    */
 483   function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 484     if (empty($name)) {
 485       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 486     }
 487     else {
 488       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 489       if (!array_key_exists($name, $tempField)) {
 490         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 491       }
 492       else {
 493         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 494           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 495         );
 496       }
 497     }
 498   }
 499
 500   /**
 501    * Store parser values
 502    *
 503    * @param CRM_Core_Session $store
 504    *
 505    * @param int $mode
 506    *
 507    * @return void
 508    * @access public
 509    */
 510   function set($store, $mode = self::MODE_SUMMARY) {
 511     $store->set('fileSize', $this->_fileSize);
 512     $store->set('lineCount', $this->_lineCount);
 513     $store->set('seperator', $this->_seperator);
 514     $store->set('fields', $this->getSelectValues());
 515     $store->set('fieldTypes', $this->getSelectTypes());
 516
 517     $store->set('headerPatterns', $this->getHeaderPatterns());
 518     $store->set('dataPatterns', $this->getDataPatterns());
 519     $store->set('columnCount', $this->_activeFieldCount);
 520
 521     $store->set('totalRowCount', $this->_totalCount);
 522     $store->set('validRowCount', $this->_validCount);
 523     $store->set('invalidRowCount', $this->_invalidRowCount);
 524     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 525     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 526     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 527     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 528     $store->set('conflictRowCount', $this->_conflictCount);
 529
 530     switch ($this->_contactType) {
 531       case 'Individual':
 532         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 533         break;
 534
 535       case 'Household':
 536         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 537         break;
 538
 539       case 'Organization':
 540         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 541     }
 542
 543     if ($this->_invalidRowCount) {
 544       $store->set('errorsFileName', $this->_errorFileName);
 545     }
 546     if ($this->_conflictCount) {
 547       $store->set('conflictsFileName', $this->_conflictFileName);
 548     }
 549     if (isset($this->_rows) && !empty($this->_rows)) {
 550       $store->set('dataValues', $this->_rows);
 551     }
 552
 553     if ($this->_invalidPledgePaymentRowCount) {
 554       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 555     }
 556
 557     if ($this->_invalidSoftCreditRowCount) {
 558       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 559     }
 560
 561     if ($mode == self::MODE_IMPORT) {
 562       $store->set('duplicateRowCount', $this->_duplicateCount);
 563       if ($this->_duplicateCount) {
 564         $store->set('duplicatesFileName', $this->_duplicateFileName);
 565       }
 566     }
 567     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 568   }
 569
 570   /**
 571    * Export data to a CSV file
 572    *
 573    * @param $fileName
 574    * @param array $header
 575    * @param data $data
 576    *
 577    * @internal param string $filename
 578    * @return void
 579    * @access public
 580    */
 581   static function exportCSV($fileName, $header, $data) {
 582     $output = array();
 583     $fd = fopen($fileName, 'w');
 584
 585     foreach ($header as $key => $value) {
 586       $header[$key] = "\"$value\"";
 587     }
 588     $config = CRM_Core_Config::singleton();
 589     $output[] = implode($config->fieldSeparator, $header);
 590
 591     foreach ($data as $datum) {
 592       foreach ($datum as $key => $value) {
 593         if (isset($value[0]) && is_array($value)) {
 594           foreach ($value[0] as $k1 => $v1) {
 595             if ($k1 == 'location_type_id') {
 596               continue;
 597             }
 598             $datum[$k1] = $v1;
 599           }
 600         }
 601         else {
 602           $datum[$key] = "\"$value\"";
 603         }
 604       }
 605       $output[] = implode($config->fieldSeparator, $datum);
 606     }
 607     fwrite($fd, implode("\n", $output));
 608     fclose($fd);
 609   }
 610
 611   /**
 612    * Determines the file extension based on error code
 613    *
 614    * @var $type error code constant
 615    * @return string
 616    * @static
 617    */
 618   /**
 619    * @param error $type
 620    *
 621    * @return string
 622    */
 623   static function errorFileName($type) {
 624     $fileName = NULL;
 625     if (empty($type)) {
 626       return $fileName;
 627     }
 628
 629     $config = CRM_Core_Config::singleton();
 630     $fileName = $config->uploadDir . "sqlImport";
 631
 632     switch ($type) {
 633       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 634         $fileName .= '.softCreditErrors';
 635         break;
 636
 637       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 638         $fileName .= '.pledgePaymentErrors';
 639         break;
 640
 641       default:
 642         $fileName = parent::errorFileName($type);
 643         break;
 644     }
 645
 646     return $fileName;
 647   }
 648
 649   /**
 650    * Determines the file name based on error code
 651    *
 652    * @var $type error code constant
 653    * @return string
 654    * @static
 655    */
 656   /**
 657    * @param error $type
 658    *
 659    * @return string
 660    */
 661   static function saveFileName($type) {
 662     $fileName = NULL;
 663     if (empty($type)) {
 664       return $fileName;
 665     }
 666
 667     switch ($type) {
 668       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 669         $fileName = 'Import_Soft_Credit_Errors.csv';
 670         break;
 671
 672       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 673         $fileName = 'Import_Pledge_Payment_Errors.csv';
 674         break;
 675
 676       default:
 677         $fileName = parent::saveFileName($type);
 678         break;
 679     }
 680
 681     return $fileName;
 682   }
 683 }
 684