3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2019
33 abstract class CRM_Contribute_Import_Parser
extends CRM_Import_Parser
{
36 * Contribution-specific result codes
37 * @see CRM_Import_Parser result code constants
39 const SOFT_CREDIT
= 512, SOFT_CREDIT_ERROR
= 1024, PLEDGE_PAYMENT
= 2048, PLEDGE_PAYMENT_ERROR
= 4096;
53 * Seperator being used
56 protected $_seperator;
59 * Total number of lines in file
62 protected $_lineCount;
65 * Running total number of valid soft credit rows
68 protected $_validSoftCreditRowCount;
71 * Running total number of invalid soft credit rows
74 protected $_invalidSoftCreditRowCount;
77 * Running total number of valid pledge payment rows
80 protected $_validPledgePaymentRowCount;
83 * Running total number of invalid pledge payment rows
86 protected $_invalidPledgePaymentRowCount;
89 * Array of pledge payment error lines, bounded by MAX_ERROR
92 protected $_pledgePaymentErrors;
95 * Array of pledge payment error lines, bounded by MAX_ERROR
98 protected $_softCreditErrors;
101 * Filename of pledge payment error data
105 protected $_pledgePaymentErrorsFileName;
108 * Filename of soft credit error data
112 protected $_softCreditErrorsFileName;
115 * Whether the file has a column header or not
119 protected $_haveColumnHeader;
122 * @param string $fileName
123 * @param string $seperator
125 * @param bool $skipColumnHeader
127 * @param int $contactType
128 * @param int $onDuplicate
129 * @param int $statusID
130 * @param int $totalRowCount
139 $skipColumnHeader = FALSE,
140 $mode = self
::MODE_PREVIEW
,
141 $contactType = self
::CONTACT_INDIVIDUAL
,
142 $onDuplicate = self
::DUPLICATE_SKIP
,
144 $totalRowCount = NULL
146 if (!is_array($fileName)) {
147 CRM_Core_Error
::fatal();
149 $fileName = $fileName['name'];
151 switch ($contactType) {
152 case self
::CONTACT_INDIVIDUAL
:
153 $this->_contactType
= 'Individual';
156 case self
::CONTACT_HOUSEHOLD
:
157 $this->_contactType
= 'Household';
160 case self
::CONTACT_ORGANIZATION
:
161 $this->_contactType
= 'Organization';
166 $this->_haveColumnHeader
= $skipColumnHeader;
168 $this->_seperator
= $seperator;
170 $fd = fopen($fileName, "r");
175 $this->_lineCount
= $this->_warningCount
= $this->_validSoftCreditRowCount
= $this->_validPledgePaymentRowCount
= 0;
176 $this->_invalidRowCount
= $this->_validCount
= $this->_invalidSoftCreditRowCount
= $this->_invalidPledgePaymentRowCount
= 0;
177 $this->_totalCount
= $this->_conflictCount
= 0;
180 $this->_warnings
= [];
181 $this->_conflicts
= [];
182 $this->_pledgePaymentErrors
= [];
183 $this->_softCreditErrors
= [];
185 $this->progressImport($statusID);
186 $startTimestamp = $currTimestamp = $prevTimestamp = time();
189 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
191 if ($mode == self
::MODE_MAPFIELD
) {
195 $this->_activeFieldCount
= count($this->_activeFields
);
201 $values = fgetcsv($fd, 8192, $seperator);
206 self
::encloseScrub($values);
208 // skip column header if we're not in mapfield mode
209 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
210 $skipColumnHeader = FALSE;
214 /* trim whitespace around the values */
217 foreach ($values as $k => $v) {
218 $values[$k] = trim($v, " \t\r\n");
221 if (CRM_Utils_System
::isNull($values)) {
225 $this->_totalCount++
;
227 if ($mode == self
::MODE_MAPFIELD
) {
228 $returnCode = $this->mapField($values);
230 elseif ($mode == self
::MODE_PREVIEW
) {
231 $returnCode = $this->preview($values);
233 elseif ($mode == self
::MODE_SUMMARY
) {
234 $returnCode = $this->summary($values);
236 elseif ($mode == self
::MODE_IMPORT
) {
237 $returnCode = $this->import($onDuplicate, $values);
238 if ($statusID && (($this->_lineCount %
50) == 0)) {
239 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
243 $returnCode = self
::ERROR
;
246 // note that a line could be valid but still produce a warning
247 if ($returnCode == self
::VALID
) {
248 $this->_validCount++
;
249 if ($mode == self
::MODE_MAPFIELD
) {
250 $this->_rows
[] = $values;
251 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
255 if ($returnCode == self
::SOFT_CREDIT
) {
256 $this->_validSoftCreditRowCount++
;
257 $this->_validCount++
;
258 if ($mode == self
::MODE_MAPFIELD
) {
259 $this->_rows
[] = $values;
260 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
264 if ($returnCode == self
::PLEDGE_PAYMENT
) {
265 $this->_validPledgePaymentRowCount++
;
266 $this->_validCount++
;
267 if ($mode == self
::MODE_MAPFIELD
) {
268 $this->_rows
[] = $values;
269 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
273 if ($returnCode == self
::WARNING
) {
274 $this->_warningCount++
;
275 if ($this->_warningCount
< $this->_maxWarningCount
) {
276 $this->_warningCount
[] = $line;
280 if ($returnCode == self
::ERROR
) {
281 $this->_invalidRowCount++
;
282 $recordNumber = $this->_lineCount
;
283 if ($this->_haveColumnHeader
) {
286 array_unshift($values, $recordNumber);
287 $this->_errors
[] = $values;
290 if ($returnCode == self
::PLEDGE_PAYMENT_ERROR
) {
291 $this->_invalidPledgePaymentRowCount++
;
292 $recordNumber = $this->_lineCount
;
293 if ($this->_haveColumnHeader
) {
296 array_unshift($values, $recordNumber);
297 $this->_pledgePaymentErrors
[] = $values;
300 if ($returnCode == self
::SOFT_CREDIT_ERROR
) {
301 $this->_invalidSoftCreditRowCount++
;
302 $recordNumber = $this->_lineCount
;
303 if ($this->_haveColumnHeader
) {
306 array_unshift($values, $recordNumber);
307 $this->_softCreditErrors
[] = $values;
310 if ($returnCode == self
::CONFLICT
) {
311 $this->_conflictCount++
;
312 $recordNumber = $this->_lineCount
;
313 if ($this->_haveColumnHeader
) {
316 array_unshift($values, $recordNumber);
317 $this->_conflicts
[] = $values;
320 if ($returnCode == self
::DUPLICATE
) {
321 if ($returnCode == self
::MULTIPLE_DUPE
) {
322 /* TODO: multi-dupes should be counted apart from singles
323 * on non-skip action */
325 $this->_duplicateCount++
;
326 $recordNumber = $this->_lineCount
;
327 if ($this->_haveColumnHeader
) {
330 array_unshift($values, $recordNumber);
331 $this->_duplicates
[] = $values;
332 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
333 $this->_validCount++
;
337 // we give the derived class a way of aborting the process
338 // note that the return code could be multiple code or'ed together
339 if ($returnCode == self
::STOP
) {
343 // if we are done processing the maxNumber of lines, break
344 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
351 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
352 $customHeaders = $mapper;
354 $customfields = CRM_Core_BAO_CustomField
::getFields('Contribution');
355 foreach ($customHeaders as $key => $value) {
356 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
357 $customHeaders[$key] = $customfields[$id][0];
360 if ($this->_invalidRowCount
) {
361 // removed view url for invlaid contacts
362 $headers = array_merge([
366 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
367 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
370 if ($this->_invalidPledgePaymentRowCount
) {
371 // removed view url for invlaid contacts
372 $headers = array_merge([
376 $this->_pledgePaymentErrorsFileName
= self
::errorFileName(self
::PLEDGE_PAYMENT_ERROR
);
377 self
::exportCSV($this->_pledgePaymentErrorsFileName
, $headers, $this->_pledgePaymentErrors
);
380 if ($this->_invalidSoftCreditRowCount
) {
381 // removed view url for invlaid contacts
382 $headers = array_merge([
386 $this->_softCreditErrorsFileName
= self
::errorFileName(self
::SOFT_CREDIT_ERROR
);
387 self
::exportCSV($this->_softCreditErrorsFileName
, $headers, $this->_softCreditErrors
);
390 if ($this->_conflictCount
) {
391 $headers = array_merge([
395 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
396 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
398 if ($this->_duplicateCount
) {
399 $headers = array_merge([
401 ts('View Contribution URL'),
404 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
405 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
408 return $this->fini();
412 * Given a list of the importable field keys that the user has selected
413 * set the active fields array to this list
415 * @param array $fieldKeys mapped array of values
417 public function setActiveFields($fieldKeys) {
418 $this->_activeFieldCount
= count($fieldKeys);
419 foreach ($fieldKeys as $key) {
420 if (empty($this->_fields
[$key])) {
421 $this->_activeFields
[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
424 $this->_activeFields
[] = clone($this->_fields
[$key]);
430 * @param array $elements
432 public function setActiveFieldSoftCredit($elements) {
433 for ($i = 0; $i < count($elements); $i++
) {
434 $this->_activeFields
[$i]->_softCreditField
= $elements[$i];
439 * @param array $elements
441 public function setActiveFieldSoftCreditType($elements) {
442 for ($i = 0; $i < count($elements); $i++
) {
443 $this->_activeFields
[$i]->_softCreditType
= $elements[$i];
448 * Format the field values for input to the api.
451 * (reference ) associative array of name/value pairs
453 public function &getActiveFieldParams() {
455 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
456 if (isset($this->_activeFields
[$i]->_value
)) {
457 if (isset($this->_activeFields
[$i]->_softCreditField
)) {
458 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
459 $params[$this->_activeFields
[$i]->_name
] = [];
461 $params[$this->_activeFields
[$i]->_name
][$i][$this->_activeFields
[$i]->_softCreditField
] = $this->_activeFields
[$i]->_value
;
462 if (isset($this->_activeFields
[$i]->_softCreditType
)) {
463 $params[$this->_activeFields
[$i]->_name
][$i]['soft_credit_type_id'] = $this->_activeFields
[$i]->_softCreditType
;
467 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
468 if (!isset($this->_activeFields
[$i]->_softCreditField
)) {
469 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
478 * @param string $name
481 * @param string $headerPattern
482 * @param string $dataPattern
484 public function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
486 $this->_fields
['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
489 $tempField = CRM_Contact_BAO_Contact
::importableFields('All', NULL);
490 if (!array_key_exists($name, $tempField)) {
491 $this->_fields
[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
494 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
495 CRM_Utils_Array
::value('hasLocationType', $tempField[$name])
502 * Store parser values.
504 * @param CRM_Core_Session $store
508 public function set($store, $mode = self
::MODE_SUMMARY
) {
509 $store->set('fileSize', $this->_fileSize
);
510 $store->set('lineCount', $this->_lineCount
);
511 $store->set('seperator', $this->_seperator
);
512 $store->set('fields', $this->getSelectValues());
513 $store->set('fieldTypes', $this->getSelectTypes());
515 $store->set('headerPatterns', $this->getHeaderPatterns());
516 $store->set('dataPatterns', $this->getDataPatterns());
517 $store->set('columnCount', $this->_activeFieldCount
);
519 $store->set('totalRowCount', $this->_totalCount
);
520 $store->set('validRowCount', $this->_validCount
);
521 $store->set('invalidRowCount', $this->_invalidRowCount
);
522 $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount
);
523 $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount
);
524 $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount
);
525 $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount
);
526 $store->set('conflictRowCount', $this->_conflictCount
);
528 switch ($this->_contactType
) {
530 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
534 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
538 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
541 if ($this->_invalidRowCount
) {
542 $store->set('errorsFileName', $this->_errorFileName
);
544 if ($this->_conflictCount
) {
545 $store->set('conflictsFileName', $this->_conflictFileName
);
547 if (isset($this->_rows
) && !empty($this->_rows
)) {
548 $store->set('dataValues', $this->_rows
);
551 if ($this->_invalidPledgePaymentRowCount
) {
552 $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName
);
555 if ($this->_invalidSoftCreditRowCount
) {
556 $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName
);
559 if ($mode == self
::MODE_IMPORT
) {
560 $store->set('duplicateRowCount', $this->_duplicateCount
);
561 if ($this->_duplicateCount
) {
562 $store->set('duplicatesFileName', $this->_duplicateFileName
);
568 * Export data to a CSV file.
570 * @param string $fileName
571 * @param array $header
574 public static function exportCSV($fileName, $header, $data) {
576 $fd = fopen($fileName, 'w');
578 foreach ($header as $key => $value) {
579 $header[$key] = "\"$value\"";
581 $config = CRM_Core_Config
::singleton();
582 $output[] = implode($config->fieldSeparator
, $header);
584 foreach ($data as $datum) {
585 foreach ($datum as $key => $value) {
586 if (isset($value[0]) && is_array($value)) {
587 foreach ($value[0] as $k1 => $v1) {
588 if ($k1 == 'location_type_id') {
595 $datum[$key] = "\"$value\"";
598 $output[] = implode($config->fieldSeparator
, $datum);
600 fwrite($fd, implode("\n", $output));
605 * Determines the file extension based on error code.
608 * Error code constant.
612 public static function errorFileName($type) {
618 $config = CRM_Core_Config
::singleton();
619 $fileName = $config->uploadDir
. "sqlImport";
622 case CRM_Contribute_Import_Parser
::SOFT_CREDIT_ERROR
:
623 $fileName .= '.softCreditErrors';
626 case CRM_Contribute_Import_Parser
::PLEDGE_PAYMENT_ERROR
:
627 $fileName .= '.pledgePaymentErrors';
631 $fileName = parent
::errorFileName($type);
639 * Determines the file name based on error code.
642 * Error code constant.
646 public static function saveFileName($type) {
653 case CRM_Contribute_Import_Parser
::SOFT_CREDIT_ERROR
:
654 $fileName = 'Import_Soft_Credit_Errors.csv';
657 case CRM_Contribute_Import_Parser
::PLEDGE_PAYMENT_ERROR
:
658 $fileName = 'Import_Pledge_Payment_Errors.csv';
662 $fileName = parent
::saveFileName($type);