3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
17 abstract class CRM_Contribute_Import_Parser
extends CRM_Import_Parser
{
20 * Contribution-specific result codes
21 * @see CRM_Import_Parser result code constants
23 const SOFT_CREDIT
= 512, SOFT_CREDIT_ERROR
= 1024, PLEDGE_PAYMENT
= 2048, PLEDGE_PAYMENT_ERROR
= 4096;
37 * Seperator being used
40 protected $_seperator;
43 * Total number of lines in file
46 protected $_lineCount;
49 * Running total number of valid soft credit rows
52 protected $_validSoftCreditRowCount;
55 * Running total number of invalid soft credit rows
58 protected $_invalidSoftCreditRowCount;
61 * Running total number of valid pledge payment rows
64 protected $_validPledgePaymentRowCount;
67 * Running total number of invalid pledge payment rows
70 protected $_invalidPledgePaymentRowCount;
73 * Array of pledge payment error lines, bounded by MAX_ERROR
76 protected $_pledgePaymentErrors;
79 * Array of pledge payment error lines, bounded by MAX_ERROR
82 protected $_softCreditErrors;
85 * Filename of pledge payment error data
89 protected $_pledgePaymentErrorsFileName;
92 * Filename of soft credit error data
96 protected $_softCreditErrorsFileName;
99 * Whether the file has a column header or not
103 protected $_haveColumnHeader;
106 * @param string $fileName
107 * @param string $seperator
109 * @param bool $skipColumnHeader
111 * @param int $contactType
112 * @param int $onDuplicate
113 * @param int $statusID
114 * @param int $totalRowCount
123 $skipColumnHeader = FALSE,
124 $mode = self
::MODE_PREVIEW
,
125 $contactType = self
::CONTACT_INDIVIDUAL
,
126 $onDuplicate = self
::DUPLICATE_SKIP
,
128 $totalRowCount = NULL
130 if (!is_array($fileName)) {
131 CRM_Core_Error
::fatal();
133 $fileName = $fileName['name'];
135 switch ($contactType) {
136 case self
::CONTACT_INDIVIDUAL
:
137 $this->_contactType
= 'Individual';
140 case self
::CONTACT_HOUSEHOLD
:
141 $this->_contactType
= 'Household';
144 case self
::CONTACT_ORGANIZATION
:
145 $this->_contactType
= 'Organization';
150 $this->_haveColumnHeader
= $skipColumnHeader;
152 $this->_seperator
= $seperator;
154 $fd = fopen($fileName, "r");
159 $this->_lineCount
= $this->_warningCount
= $this->_validSoftCreditRowCount
= $this->_validPledgePaymentRowCount
= 0;
160 $this->_invalidRowCount
= $this->_validCount
= $this->_invalidSoftCreditRowCount
= $this->_invalidPledgePaymentRowCount
= 0;
161 $this->_totalCount
= $this->_conflictCount
= 0;
164 $this->_warnings
= [];
165 $this->_conflicts
= [];
166 $this->_pledgePaymentErrors
= [];
167 $this->_softCreditErrors
= [];
169 $this->progressImport($statusID);
170 $startTimestamp = $currTimestamp = $prevTimestamp = time();
173 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
175 if ($mode == self
::MODE_MAPFIELD
) {
179 $this->_activeFieldCount
= count($this->_activeFields
);
185 $values = fgetcsv($fd, 8192, $seperator);
190 self
::encloseScrub($values);
192 // skip column header if we're not in mapfield mode
193 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
194 $skipColumnHeader = FALSE;
198 /* trim whitespace around the values */
201 foreach ($values as $k => $v) {
202 $values[$k] = trim($v, " \t\r\n");
205 if (CRM_Utils_System
::isNull($values)) {
209 $this->_totalCount++
;
211 if ($mode == self
::MODE_MAPFIELD
) {
212 $returnCode = $this->mapField($values);
214 elseif ($mode == self
::MODE_PREVIEW
) {
215 $returnCode = $this->preview($values);
217 elseif ($mode == self
::MODE_SUMMARY
) {
218 $returnCode = $this->summary($values);
220 elseif ($mode == self
::MODE_IMPORT
) {
221 $returnCode = $this->import($onDuplicate, $values);
222 if ($statusID && (($this->_lineCount %
50) == 0)) {
223 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
227 $returnCode = self
::ERROR
;
230 // note that a line could be valid but still produce a warning
231 if ($returnCode == self
::VALID
) {
232 $this->_validCount++
;
233 if ($mode == self
::MODE_MAPFIELD
) {
234 $this->_rows
[] = $values;
235 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
239 if ($returnCode == self
::SOFT_CREDIT
) {
240 $this->_validSoftCreditRowCount++
;
241 $this->_validCount++
;
242 if ($mode == self
::MODE_MAPFIELD
) {
243 $this->_rows
[] = $values;
244 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
248 if ($returnCode == self
::PLEDGE_PAYMENT
) {
249 $this->_validPledgePaymentRowCount++
;
250 $this->_validCount++
;
251 if ($mode == self
::MODE_MAPFIELD
) {
252 $this->_rows
[] = $values;
253 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
257 if ($returnCode == self
::WARNING
) {
258 $this->_warningCount++
;
259 if ($this->_warningCount
< $this->_maxWarningCount
) {
260 $this->_warningCount
[] = $line;
264 if ($returnCode == self
::ERROR
) {
265 $this->_invalidRowCount++
;
266 $recordNumber = $this->_lineCount
;
267 if ($this->_haveColumnHeader
) {
270 array_unshift($values, $recordNumber);
271 $this->_errors
[] = $values;
274 if ($returnCode == self
::PLEDGE_PAYMENT_ERROR
) {
275 $this->_invalidPledgePaymentRowCount++
;
276 $recordNumber = $this->_lineCount
;
277 if ($this->_haveColumnHeader
) {
280 array_unshift($values, $recordNumber);
281 $this->_pledgePaymentErrors
[] = $values;
284 if ($returnCode == self
::SOFT_CREDIT_ERROR
) {
285 $this->_invalidSoftCreditRowCount++
;
286 $recordNumber = $this->_lineCount
;
287 if ($this->_haveColumnHeader
) {
290 array_unshift($values, $recordNumber);
291 $this->_softCreditErrors
[] = $values;
294 if ($returnCode == self
::CONFLICT
) {
295 $this->_conflictCount++
;
296 $recordNumber = $this->_lineCount
;
297 if ($this->_haveColumnHeader
) {
300 array_unshift($values, $recordNumber);
301 $this->_conflicts
[] = $values;
304 if ($returnCode == self
::DUPLICATE
) {
305 if ($returnCode == self
::MULTIPLE_DUPE
) {
306 /* TODO: multi-dupes should be counted apart from singles
307 * on non-skip action */
309 $this->_duplicateCount++
;
310 $recordNumber = $this->_lineCount
;
311 if ($this->_haveColumnHeader
) {
314 array_unshift($values, $recordNumber);
315 $this->_duplicates
[] = $values;
316 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
317 $this->_validCount++
;
321 // we give the derived class a way of aborting the process
322 // note that the return code could be multiple code or'ed together
323 if ($returnCode == self
::STOP
) {
327 // if we are done processing the maxNumber of lines, break
328 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
335 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
336 $customHeaders = $mapper;
338 $customfields = CRM_Core_BAO_CustomField
::getFields('Contribution');
339 foreach ($customHeaders as $key => $value) {
340 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
341 $customHeaders[$key] = $customfields[$id][0];
344 if ($this->_invalidRowCount
) {
345 // removed view url for invlaid contacts
346 $headers = array_merge([
350 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
351 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
354 if ($this->_invalidPledgePaymentRowCount
) {
355 // removed view url for invlaid contacts
356 $headers = array_merge([
360 $this->_pledgePaymentErrorsFileName
= self
::errorFileName(self
::PLEDGE_PAYMENT_ERROR
);
361 self
::exportCSV($this->_pledgePaymentErrorsFileName
, $headers, $this->_pledgePaymentErrors
);
364 if ($this->_invalidSoftCreditRowCount
) {
365 // removed view url for invlaid contacts
366 $headers = array_merge([
370 $this->_softCreditErrorsFileName
= self
::errorFileName(self
::SOFT_CREDIT_ERROR
);
371 self
::exportCSV($this->_softCreditErrorsFileName
, $headers, $this->_softCreditErrors
);
374 if ($this->_conflictCount
) {
375 $headers = array_merge([
379 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
380 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
382 if ($this->_duplicateCount
) {
383 $headers = array_merge([
385 ts('View Contribution URL'),
388 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
389 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
392 return $this->fini();
396 * Given a list of the importable field keys that the user has selected
397 * set the active fields array to this list
399 * @param array $fieldKeys mapped array of values
401 public function setActiveFields($fieldKeys) {
402 $this->_activeFieldCount
= count($fieldKeys);
403 foreach ($fieldKeys as $key) {
404 if (empty($this->_fields
[$key])) {
405 $this->_activeFields
[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
408 $this->_activeFields
[] = clone($this->_fields
[$key]);
414 * Store the soft credit field information.
416 * This was perhaps done this way on the believe that a lot of code pain
417 * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise
418 * readability & maintainability next since we can just work with functions to retrieve
419 * data from the metadata.
421 * @param array $elements
423 public function setActiveFieldSoftCredit($elements) {
424 foreach ((array) $elements as $i => $element) {
425 $this->_activeFields
[$i]->_softCreditField
= $element;
430 * Store the soft credit field type information.
432 * This was perhaps done this way on the believe that a lot of code pain
433 * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise
434 * readability & maintainability next since we can just work with functions to retrieve
435 * data from the metadata.
437 * @param array $elements
439 public function setActiveFieldSoftCreditType($elements) {
440 foreach ((array) $elements as $i => $element) {
441 $this->_activeFields
[$i]->_softCreditType
= $element;
446 * Format the field values for input to the api.
449 * (reference ) associative array of name/value pairs
451 public function &getActiveFieldParams() {
453 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
454 if (isset($this->_activeFields
[$i]->_value
)) {
455 if (isset($this->_activeFields
[$i]->_softCreditField
)) {
456 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
457 $params[$this->_activeFields
[$i]->_name
] = [];
459 $params[$this->_activeFields
[$i]->_name
][$i][$this->_activeFields
[$i]->_softCreditField
] = $this->_activeFields
[$i]->_value
;
460 if (isset($this->_activeFields
[$i]->_softCreditType
)) {
461 $params[$this->_activeFields
[$i]->_name
][$i]['soft_credit_type_id'] = $this->_activeFields
[$i]->_softCreditType
;
465 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
466 if (!isset($this->_activeFields
[$i]->_softCreditField
)) {
467 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
476 * @param string $name
479 * @param string $headerPattern
480 * @param string $dataPattern
482 public function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
484 $this->_fields
['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
487 $tempField = CRM_Contact_BAO_Contact
::importableFields('All', NULL);
488 if (!array_key_exists($name, $tempField)) {
489 $this->_fields
[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
492 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
493 CRM_Utils_Array
::value('hasLocationType', $tempField[$name])
500 * Store parser values.
502 * @param CRM_Core_Session $store
506 public function set($store, $mode = self
::MODE_SUMMARY
) {
507 $store->set('fileSize', $this->_fileSize
);
508 $store->set('lineCount', $this->_lineCount
);
509 $store->set('seperator', $this->_seperator
);
510 $store->set('fields', $this->getSelectValues());
511 $store->set('fieldTypes', $this->getSelectTypes());
513 $store->set('headerPatterns', $this->getHeaderPatterns());
514 $store->set('dataPatterns', $this->getDataPatterns());
515 $store->set('columnCount', $this->_activeFieldCount
);
517 $store->set('totalRowCount', $this->_totalCount
);
518 $store->set('validRowCount', $this->_validCount
);
519 $store->set('invalidRowCount', $this->_invalidRowCount
);
520 $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount
);
521 $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount
);
522 $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount
);
523 $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount
);
524 $store->set('conflictRowCount', $this->_conflictCount
);
526 switch ($this->_contactType
) {
528 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
532 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
536 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
539 if ($this->_invalidRowCount
) {
540 $store->set('errorsFileName', $this->_errorFileName
);
542 if ($this->_conflictCount
) {
543 $store->set('conflictsFileName', $this->_conflictFileName
);
545 if (isset($this->_rows
) && !empty($this->_rows
)) {
546 $store->set('dataValues', $this->_rows
);
549 if ($this->_invalidPledgePaymentRowCount
) {
550 $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName
);
553 if ($this->_invalidSoftCreditRowCount
) {
554 $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName
);
557 if ($mode == self
::MODE_IMPORT
) {
558 $store->set('duplicateRowCount', $this->_duplicateCount
);
559 if ($this->_duplicateCount
) {
560 $store->set('duplicatesFileName', $this->_duplicateFileName
);
566 * Export data to a CSV file.
568 * @param string $fileName
569 * @param array $header
572 public static function exportCSV($fileName, $header, $data) {
574 $fd = fopen($fileName, 'w');
576 foreach ($header as $key => $value) {
577 $header[$key] = "\"$value\"";
579 $config = CRM_Core_Config
::singleton();
580 $output[] = implode($config->fieldSeparator
, $header);
582 foreach ($data as $datum) {
583 foreach ($datum as $key => $value) {
584 if (isset($value[0]) && is_array($value)) {
585 foreach ($value[0] as $k1 => $v1) {
586 if ($k1 == 'location_type_id') {
593 $datum[$key] = "\"$value\"";
596 $output[] = implode($config->fieldSeparator
, $datum);
598 fwrite($fd, implode("\n", $output));
603 * Determines the file extension based on error code.
606 * Error code constant.
610 public static function errorFileName($type) {
616 $config = CRM_Core_Config
::singleton();
617 $fileName = $config->uploadDir
. "sqlImport";
620 case CRM_Contribute_Import_Parser
::SOFT_CREDIT_ERROR
:
621 $fileName .= '.softCreditErrors';
624 case CRM_Contribute_Import_Parser
::PLEDGE_PAYMENT_ERROR
:
625 $fileName .= '.pledgePaymentErrors';
629 $fileName = parent
::errorFileName($type);
637 * Determines the file name based on error code.
640 * Error code constant.
644 public static function saveFileName($type) {
651 case CRM_Contribute_Import_Parser
::SOFT_CREDIT_ERROR
:
652 $fileName = 'Import_Soft_Credit_Errors.csv';
655 case CRM_Contribute_Import_Parser
::PLEDGE_PAYMENT_ERROR
:
656 $fileName = 'Import_Pledge_Payment_Errors.csv';
660 $fileName = parent
::saveFileName($type);