CRM/Contribute/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2017                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2017
  32  */
  33 abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser {
  34
  35   /**
  36    * Contribution-specific result codes
  37    * @see CRM_Import_Parser result code constants
  38    */
  39   const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096;
  40
  41   protected $_fileName;
  42
  43   /**
  44    * Imported file size
  45    */
  46   protected $_fileSize;
  47
  48   /**
  49    * Seperator being used
  50    */
  51   protected $_seperator;
  52
  53   /**
  54    * Total number of lines in file
  55    */
  56   protected $_lineCount;
  57
  58   /**
  59    * Running total number of valid soft credit rows
  60    */
  61   protected $_validSoftCreditRowCount;
  62
  63   /**
  64    * Running total number of invalid soft credit rows
  65    */
  66   protected $_invalidSoftCreditRowCount;
  67
  68   /**
  69    * Running total number of valid pledge payment rows
  70    */
  71   protected $_validPledgePaymentRowCount;
  72
  73   /**
  74    * Running total number of invalid pledge payment rows
  75    */
  76   protected $_invalidPledgePaymentRowCount;
  77
  78   /**
  79    * Array of pledge payment error lines, bounded by MAX_ERROR
  80    */
  81   protected $_pledgePaymentErrors;
  82
  83   /**
  84    * Array of pledge payment error lines, bounded by MAX_ERROR
  85    */
  86   protected $_softCreditErrors;
  87
  88   /**
  89    * Filename of pledge payment error data
  90    *
  91    * @var string
  92    */
  93   protected $_pledgePaymentErrorsFileName;
  94
  95   /**
  96    * Filename of soft credit error data
  97    *
  98    * @var string
  99    */
 100   protected $_softCreditErrorsFileName;
 101
 102   /**
 103    * Whether the file has a column header or not
 104    *
 105    * @var boolean
 106    */
 107   protected $_haveColumnHeader;
 108
 109   /**
 110    * @param string $fileName
 111    * @param string $seperator
 112    * @param $mapper
 113    * @param bool $skipColumnHeader
 114    * @param int $mode
 115    * @param int $contactType
 116    * @param int $onDuplicate
 117    *
 118    * @return mixed
 119    * @throws Exception
 120    */
 121   public function run(
 122     $fileName,
 123     $seperator = ',',
 124     &$mapper,
 125     $skipColumnHeader = FALSE,
 126     $mode = self::MODE_PREVIEW,
 127     $contactType = self::CONTACT_INDIVIDUAL,
 128     $onDuplicate = self::DUPLICATE_SKIP
 129   ) {
 130     if (!is_array($fileName)) {
 131       CRM_Core_Error::fatal();
 132     }
 133     $fileName = $fileName['name'];
 134
 135     switch ($contactType) {
 136       case self::CONTACT_INDIVIDUAL:
 137         $this->_contactType = 'Individual';
 138         break;
 139
 140       case self::CONTACT_HOUSEHOLD:
 141         $this->_contactType = 'Household';
 142         break;
 143
 144       case self::CONTACT_ORGANIZATION:
 145         $this->_contactType = 'Organization';
 146     }
 147
 148     $this->init();
 149
 150     $this->_haveColumnHeader = $skipColumnHeader;
 151
 152     $this->_seperator = $seperator;
 153
 154     $fd = fopen($fileName, "r");
 155     if (!$fd) {
 156       return FALSE;
 157     }
 158
 159     $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0;
 160     $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0;
 161     $this->_totalCount = $this->_conflictCount = 0;
 162
 163     $this->_errors = array();
 164     $this->_warnings = array();
 165     $this->_conflicts = array();
 166     $this->_pledgePaymentErrors = array();
 167     $this->_softCreditErrors = array();
 168
 169     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 170
 171     if ($mode == self::MODE_MAPFIELD) {
 172       $this->_rows = array();
 173     }
 174     else {
 175       $this->_activeFieldCount = count($this->_activeFields);
 176     }
 177
 178     while (!feof($fd)) {
 179       $this->_lineCount++;
 180
 181       $values = fgetcsv($fd, 8192, $seperator);
 182       if (!$values) {
 183         continue;
 184       }
 185
 186       self::encloseScrub($values);
 187
 188       // skip column header if we're not in mapfield mode
 189       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 190         $skipColumnHeader = FALSE;
 191         continue;
 192       }
 193
 194       /* trim whitespace around the values */
 195
 196       $empty = TRUE;
 197       foreach ($values as $k => $v) {
 198         $values[$k] = trim($v, " \t\r\n");
 199       }
 200
 201       if (CRM_Utils_System::isNull($values)) {
 202         continue;
 203       }
 204
 205       $this->_totalCount++;
 206
 207       if ($mode == self::MODE_MAPFIELD) {
 208         $returnCode = $this->mapField($values);
 209       }
 210       elseif ($mode == self::MODE_PREVIEW) {
 211         $returnCode = $this->preview($values);
 212       }
 213       elseif ($mode == self::MODE_SUMMARY) {
 214         $returnCode = $this->summary($values);
 215       }
 216       elseif ($mode == self::MODE_IMPORT) {
 217         $returnCode = $this->import($onDuplicate, $values);
 218       }
 219       else {
 220         $returnCode = self::ERROR;
 221       }
 222
 223       // note that a line could be valid but still produce a warning
 224       if ($returnCode == self::VALID) {
 225         $this->_validCount++;
 226         if ($mode == self::MODE_MAPFIELD) {
 227           $this->_rows[] = $values;
 228           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 229         }
 230       }
 231
 232       if ($returnCode == self::SOFT_CREDIT) {
 233         $this->_validSoftCreditRowCount++;
 234         $this->_validCount++;
 235         if ($mode == self::MODE_MAPFIELD) {
 236           $this->_rows[] = $values;
 237           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 238         }
 239       }
 240
 241       if ($returnCode == self::PLEDGE_PAYMENT) {
 242         $this->_validPledgePaymentRowCount++;
 243         $this->_validCount++;
 244         if ($mode == self::MODE_MAPFIELD) {
 245           $this->_rows[] = $values;
 246           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 247         }
 248       }
 249
 250       if ($returnCode == self::WARNING) {
 251         $this->_warningCount++;
 252         if ($this->_warningCount < $this->_maxWarningCount) {
 253           $this->_warningCount[] = $line;
 254         }
 255       }
 256
 257       if ($returnCode == self::ERROR) {
 258         $this->_invalidRowCount++;
 259         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 260           $recordNumber = $this->_lineCount;
 261           if ($this->_haveColumnHeader) {
 262             $recordNumber--;
 263           }
 264           array_unshift($values, $recordNumber);
 265           $this->_errors[] = $values;
 266         }
 267       }
 268
 269       if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
 270         $this->_invalidPledgePaymentRowCount++;
 271         if ($this->_invalidPledgePaymentRowCount < $this->_maxErrorCount) {
 272           $recordNumber = $this->_lineCount;
 273           if ($this->_haveColumnHeader) {
 274             $recordNumber--;
 275           }
 276           array_unshift($values, $recordNumber);
 277           $this->_pledgePaymentErrors[] = $values;
 278         }
 279       }
 280
 281       if ($returnCode == self::SOFT_CREDIT_ERROR) {
 282         $this->_invalidSoftCreditRowCount++;
 283         if ($this->_invalidSoftCreditRowCount < $this->_maxErrorCount) {
 284           $recordNumber = $this->_lineCount;
 285           if ($this->_haveColumnHeader) {
 286             $recordNumber--;
 287           }
 288           array_unshift($values, $recordNumber);
 289           $this->_softCreditErrors[] = $values;
 290         }
 291       }
 292
 293       if ($returnCode == self::CONFLICT) {
 294         $this->_conflictCount++;
 295         $recordNumber = $this->_lineCount;
 296         if ($this->_haveColumnHeader) {
 297           $recordNumber--;
 298         }
 299         array_unshift($values, $recordNumber);
 300         $this->_conflicts[] = $values;
 301       }
 302
 303       if ($returnCode == self::DUPLICATE) {
 304         if ($returnCode == self::MULTIPLE_DUPE) {
 305           /* TODO: multi-dupes should be counted apart from singles
 306            * on non-skip action */
 307         }
 308         $this->_duplicateCount++;
 309         $recordNumber = $this->_lineCount;
 310         if ($this->_haveColumnHeader) {
 311           $recordNumber--;
 312         }
 313         array_unshift($values, $recordNumber);
 314         $this->_duplicates[] = $values;
 315         if ($onDuplicate != self::DUPLICATE_SKIP) {
 316           $this->_validCount++;
 317         }
 318       }
 319
 320       // we give the derived class a way of aborting the process
 321       // note that the return code could be multiple code or'ed together
 322       if ($returnCode == self::STOP) {
 323         break;
 324       }
 325
 326       // if we are done processing the maxNumber of lines, break
 327       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 328         break;
 329       }
 330     }
 331
 332     fclose($fd);
 333
 334     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 335       $customHeaders = $mapper;
 336
 337       $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
 338       foreach ($customHeaders as $key => $value) {
 339         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 340           $customHeaders[$key] = $customfields[$id][0];
 341         }
 342       }
 343       if ($this->_invalidRowCount) {
 344         // removed view url for invlaid contacts
 345         $headers = array_merge(array(
 346             ts('Line Number'),
 347             ts('Reason'),
 348           ),
 349           $customHeaders
 350         );
 351         $this->_errorFileName = self::errorFileName(self::ERROR);
 352         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 353       }
 354
 355       if ($this->_invalidPledgePaymentRowCount) {
 356         // removed view url for invlaid contacts
 357         $headers = array_merge(array(
 358             ts('Line Number'),
 359             ts('Reason'),
 360           ),
 361           $customHeaders
 362         );
 363         $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
 364         self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
 365       }
 366
 367       if ($this->_invalidSoftCreditRowCount) {
 368         // removed view url for invlaid contacts
 369         $headers = array_merge(array(
 370             ts('Line Number'),
 371             ts('Reason'),
 372           ),
 373           $customHeaders
 374         );
 375         $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
 376         self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
 377       }
 378
 379       if ($this->_conflictCount) {
 380         $headers = array_merge(array(
 381             ts('Line Number'),
 382             ts('Reason'),
 383           ),
 384           $customHeaders
 385         );
 386         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 387         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 388       }
 389       if ($this->_duplicateCount) {
 390         $headers = array_merge(array(
 391             ts('Line Number'),
 392             ts('View Contribution URL'),
 393           ),
 394           $customHeaders
 395         );
 396
 397         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 398         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 399       }
 400     }
 401     return $this->fini();
 402   }
 403
 404   /**
 405    * Given a list of the importable field keys that the user has selected
 406    * set the active fields array to this list
 407    *
 408    * @param array $fieldKeys mapped array of values
 409    */
 410   public function setActiveFields($fieldKeys) {
 411     $this->_activeFieldCount = count($fieldKeys);
 412     foreach ($fieldKeys as $key) {
 413       if (empty($this->_fields[$key])) {
 414         $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
 415       }
 416       else {
 417         $this->_activeFields[] = clone($this->_fields[$key]);
 418       }
 419     }
 420   }
 421
 422   /**
 423    * @param array $elements
 424    */
 425   public function setActiveFieldSoftCredit($elements) {
 426     for ($i = 0; $i < count($elements); $i++) {
 427       $this->_activeFields[$i]->_softCreditField = $elements[$i];
 428     }
 429   }
 430
 431   /**
 432    * @param array $elements
 433    */
 434   public function setActiveFieldSoftCreditType($elements) {
 435     for ($i = 0; $i < count($elements); $i++) {
 436       $this->_activeFields[$i]->_softCreditType = $elements[$i];
 437     }
 438   }
 439
 440   /**
 441    * Format the field values for input to the api.
 442    *
 443    * @return array
 444    *   (reference ) associative array of name/value pairs
 445    */
 446   public function &getActiveFieldParams() {
 447     $params = array();
 448     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 449       if (isset($this->_activeFields[$i]->_value)) {
 450         if (isset($this->_activeFields[$i]->_softCreditField)) {
 451           if (!isset($params[$this->_activeFields[$i]->_name])) {
 452             $params[$this->_activeFields[$i]->_name] = array();
 453           }
 454           $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value;
 455           if (isset($this->_activeFields[$i]->_softCreditType)) {
 456             $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType;
 457           }
 458         }
 459
 460         if (!isset($params[$this->_activeFields[$i]->_name])) {
 461           if (!isset($this->_activeFields[$i]->_softCreditField)) {
 462             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 463           }
 464         }
 465       }
 466     }
 467     return $params;
 468   }
 469
 470   /**
 471    * @param string $name
 472    * @param $title
 473    * @param int $type
 474    * @param string $headerPattern
 475    * @param string $dataPattern
 476    */
 477   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 478     if (empty($name)) {
 479       $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 480     }
 481     else {
 482       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 483       if (!array_key_exists($name, $tempField)) {
 484         $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 485       }
 486       else {
 487         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 488           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 489         );
 490       }
 491     }
 492   }
 493
 494   /**
 495    * Store parser values.
 496    *
 497    * @param CRM_Core_Session $store
 498    *
 499    * @param int $mode
 500    */
 501   public function set($store, $mode = self::MODE_SUMMARY) {
 502     $store->set('fileSize', $this->_fileSize);
 503     $store->set('lineCount', $this->_lineCount);
 504     $store->set('seperator', $this->_seperator);
 505     $store->set('fields', $this->getSelectValues());
 506     $store->set('fieldTypes', $this->getSelectTypes());
 507
 508     $store->set('headerPatterns', $this->getHeaderPatterns());
 509     $store->set('dataPatterns', $this->getDataPatterns());
 510     $store->set('columnCount', $this->_activeFieldCount);
 511
 512     $store->set('totalRowCount', $this->_totalCount);
 513     $store->set('validRowCount', $this->_validCount);
 514     $store->set('invalidRowCount', $this->_invalidRowCount);
 515     $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount);
 516     $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount);
 517     $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount);
 518     $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount);
 519     $store->set('conflictRowCount', $this->_conflictCount);
 520
 521     switch ($this->_contactType) {
 522       case 'Individual':
 523         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 524         break;
 525
 526       case 'Household':
 527         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 528         break;
 529
 530       case 'Organization':
 531         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 532     }
 533
 534     if ($this->_invalidRowCount) {
 535       $store->set('errorsFileName', $this->_errorFileName);
 536     }
 537     if ($this->_conflictCount) {
 538       $store->set('conflictsFileName', $this->_conflictFileName);
 539     }
 540     if (isset($this->_rows) && !empty($this->_rows)) {
 541       $store->set('dataValues', $this->_rows);
 542     }
 543
 544     if ($this->_invalidPledgePaymentRowCount) {
 545       $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName);
 546     }
 547
 548     if ($this->_invalidSoftCreditRowCount) {
 549       $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName);
 550     }
 551
 552     if ($mode == self::MODE_IMPORT) {
 553       $store->set('duplicateRowCount', $this->_duplicateCount);
 554       if ($this->_duplicateCount) {
 555         $store->set('duplicatesFileName', $this->_duplicateFileName);
 556       }
 557     }
 558   }
 559
 560   /**
 561    * Export data to a CSV file.
 562    *
 563    * @param string $fileName
 564    * @param array $header
 565    * @param array $data
 566    */
 567   public static function exportCSV($fileName, $header, $data) {
 568     $output = array();
 569     $fd = fopen($fileName, 'w');
 570
 571     foreach ($header as $key => $value) {
 572       $header[$key] = "\"$value\"";
 573     }
 574     $config = CRM_Core_Config::singleton();
 575     $output[] = implode($config->fieldSeparator, $header);
 576
 577     foreach ($data as $datum) {
 578       foreach ($datum as $key => $value) {
 579         if (isset($value[0]) && is_array($value)) {
 580           foreach ($value[0] as $k1 => $v1) {
 581             if ($k1 == 'location_type_id') {
 582               continue;
 583             }
 584             $datum[$k1] = $v1;
 585           }
 586         }
 587         else {
 588           $datum[$key] = "\"$value\"";
 589         }
 590       }
 591       $output[] = implode($config->fieldSeparator, $datum);
 592     }
 593     fwrite($fd, implode("\n", $output));
 594     fclose($fd);
 595   }
 596
 597   /**
 598    * Determines the file extension based on error code.
 599    *
 600    * @param int $type
 601    *   Error code constant.
 602    *
 603    * @return string
 604    */
 605   public static function errorFileName($type) {
 606     $fileName = NULL;
 607     if (empty($type)) {
 608       return $fileName;
 609     }
 610
 611     $config = CRM_Core_Config::singleton();
 612     $fileName = $config->uploadDir . "sqlImport";
 613
 614     switch ($type) {
 615       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 616         $fileName .= '.softCreditErrors';
 617         break;
 618
 619       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 620         $fileName .= '.pledgePaymentErrors';
 621         break;
 622
 623       default:
 624         $fileName = parent::errorFileName($type);
 625         break;
 626     }
 627
 628     return $fileName;
 629   }
 630
 631   /**
 632    * Determines the file name based on error code.
 633    *
 634    * @param int $type
 635    *   Error code constant.
 636    *
 637    * @return string
 638    */
 639   public static function saveFileName($type) {
 640     $fileName = NULL;
 641     if (empty($type)) {
 642       return $fileName;
 643     }
 644
 645     switch ($type) {
 646       case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR:
 647         $fileName = 'Import_Soft_Credit_Errors.csv';
 648         break;
 649
 650       case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR:
 651         $fileName = 'Import_Pledge_Payment_Errors.csv';
 652         break;
 653
 654       default:
 655         $fileName = parent::saveFileName($type);
 656         break;
 657     }
 658
 659     return $fileName;
 660   }
 661
 662 }