From 8dc9763a2d9de8cdffa311fc0f2cb7ff8572ef94 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Thu, 7 Apr 2022 10:39:09 +1200 Subject: [PATCH] Simplify class inheritance CRM_Contribute_Import_Parser_Contribution is the only class in our universe to extend CRM_Contribute_Import_Parser and this class only adds confusion to the mix as functions are 'distributed' between them. This removes the inheritance, leaving only the constant that is still used in the now-deprecated class --- .toxic.json | 2 +- CRM/Contribute/Import/Form/Preview.php | 12 +- CRM/Contribute/Import/Form/Summary.php | 8 +- CRM/Contribute/Import/Parser.php | 667 ------------------ CRM/Contribute/Import/Parser/Contribution.php | 662 ++++++++++++++++- .../Import/Parser/ContributionTest.php | 2 +- 6 files changed, 667 insertions(+), 686 deletions(-) delete mode 100644 CRM/Contribute/Import/Parser.php diff --git a/.toxic.json b/.toxic.json index d24776f283..f5f5c6fb27 100644 --- a/.toxic.json +++ b/.toxic.json @@ -32,7 +32,7 @@ "CRM_Contribute_Form_Contribution_Main::formRule()": "toxicAlert", "CRM_Contribute_Form_Contribution_Main::submit()": "toxicAlert", "CRM_Contribute_Form_Task_Invoice::printPDF()": "toxicAlert", - "CRM_Contribute_Import_Parser::run()": "toxicAlert", + "CRM_Contribute_Import_Parser_Contribution::run()": "toxicAlert", "CRM_Core_BAO_ActionScheduleTest::setUp()": "toxicAlert", "CRM_Core_BAO_CustomField::formatCustomField()": "toxicAlert", "CRM_Core_BAO_CustomGroup::getTree()": "toxicAlert", diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index e45de43a25..9a6ba068de 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -54,17 +54,17 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { } if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($conflictRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } @@ -160,11 +160,11 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { fclose($fd); $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } } diff --git a/CRM/Contribute/Import/Form/Summary.php b/CRM/Contribute/Import/Form/Summary.php index 4d86b06058..6714befabc 100644 --- a/CRM/Contribute/Import/Form/Summary.php +++ b/CRM/Contribute/Import/Form/Summary.php @@ -35,13 +35,13 @@ class CRM_Contribute_Import_Form_Summary extends CRM_Import_Form_Summary { $invalidRowCount = $this->get('invalidRowCount'); $invalidSoftCreditRowCount = $this->get('invalidSoftCreditRowCount'); if ($invalidSoftCreditRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadSoftCreditErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validSoftCreditRowCount = $this->get('validSoftCreditRowCount'); $invalidPledgePaymentRowCount = $this->get('invalidPledgePaymentRowCount'); if ($invalidPledgePaymentRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadPledgePaymentErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validPledgePaymentRowCount = $this->get('validPledgePaymentRowCount'); @@ -50,11 +50,11 @@ class CRM_Contribute_Import_Form_Summary extends CRM_Import_Form_Summary { $onDuplicate = $this->get('onDuplicate'); $mismatchCount = $this->get('unMatchCount'); if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } elseif ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } else { diff --git a/CRM/Contribute/Import/Parser.php b/CRM/Contribute/Import/Parser.php deleted file mode 100644 index 8cd494f386..0000000000 --- a/CRM/Contribute/Import/Parser.php +++ /dev/null @@ -1,667 +0,0 @@ -_contactType = 'Individual'; - break; - - case self::CONTACT_HOUSEHOLD: - $this->_contactType = 'Household'; - break; - - case self::CONTACT_ORGANIZATION: - $this->_contactType = 'Organization'; - } - - $this->init(); - - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - - $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; - $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; - $this->_totalCount = $this->_conflictCount = 0; - - $this->_errors = []; - $this->_warnings = []; - $this->_conflicts = []; - $this->_pledgePaymentErrors = []; - $this->_softCreditErrors = []; - if ($statusID) { - $this->progressImport($statusID); - $startTimestamp = $currTimestamp = $prevTimestamp = time(); - } - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } - - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - - if (CRM_Utils_System::isNull($values)) { - continue; - } - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = $this->mapField($values); - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); - if ($statusID && (($this->_lineCount % 50) == 0)) { - $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); - } - } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode == self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::SOFT_CREDIT) { - $this->_validSoftCreditRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::PLEDGE_PAYMENT) { - $this->_validPledgePaymentRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::WARNING) { - $this->_warningCount++; - if ($this->_warningCount < $this->_maxWarningCount) { - $this->_warningCount[] = $line; - } - } - - if ($returnCode == self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { - $this->_invalidPledgePaymentRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_pledgePaymentErrors[] = $values; - } - - if ($returnCode == self::SOFT_CREDIT_ERROR) { - $this->_invalidSoftCreditRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_softCreditErrors[] = $values; - } - - if ($returnCode == self::CONFLICT) { - $this->_conflictCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_conflicts[] = $values; - } - - if ($returnCode == self::DUPLICATE) { - if ($returnCode == self::MULTIPLE_DUPE) { - /* TODO: multi-dupes should be counted apart from singles - * on non-skip action */ - } - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { - $this->_validCount++; - } - } - - // we give the derived class a way of aborting the process - // note that the return code could be multiple code or'ed together - if ($returnCode == self::STOP) { - break; - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - - if ($this->_invalidPledgePaymentRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); - self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); - } - - if ($this->_invalidSoftCreditRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); - self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); - } - - if ($this->_conflictCount) { - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_conflictFileName = self::errorFileName(self::CONFLICT); - self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); - } - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Contribution URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } - } - return $this->fini(); - } - - /** - * Given a list of the importable field keys that the user has selected - * set the active fields array to this list - * - * @param array $fieldKeys mapped array of values - */ - public function setActiveFields($fieldKeys) { - $this->_activeFieldCount = count($fieldKeys); - foreach ($fieldKeys as $key) { - if (empty($this->_fields[$key])) { - $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); - } - else { - $this->_activeFields[] = clone($this->_fields[$key]); - } - } - } - - /** - * Store the soft credit field information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCredit($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditField = $element; - } - } - - /** - * Store the soft credit field type information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCreditType($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditType = $element; - } - } - - /** - * Format the field values for input to the api. - * - * @return array - * (reference ) associative array of name/value pairs - */ - public function &getActiveFieldParams() { - $params = []; - for ($i = 0; $i < $this->_activeFieldCount; $i++) { - if (isset($this->_activeFields[$i]->_value)) { - if (isset($this->_activeFields[$i]->_softCreditField)) { - if (!isset($params[$this->_activeFields[$i]->_name])) { - $params[$this->_activeFields[$i]->_name] = []; - } - $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; - if (isset($this->_activeFields[$i]->_softCreditType)) { - $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; - } - } - - if (!isset($params[$this->_activeFields[$i]->_name])) { - if (!isset($this->_activeFields[$i]->_softCreditField)) { - $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; - } - } - } - } - return $params; - } - - /** - * @param string $name - * @param $title - * @param int $type - * @param string $headerPattern - * @param string $dataPattern - */ - public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { - if (empty($name)) { - $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); - if (!array_key_exists($name, $tempField)) { - $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, - CRM_Utils_Array::value('hasLocationType', $tempField[$name]) - ); - } - } - } - - /** - * Store parser values. - * - * @param CRM_Core_Session $store - * - * @param int $mode - */ - public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - $store->set('fieldTypes', $this->getSelectTypes()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); - $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); - $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); - $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); - $store->set('conflictRowCount', $this->_conflictCount); - - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - if ($this->_conflictCount) { - $store->set('conflictsFileName', $this->_conflictFileName); - } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($this->_invalidPledgePaymentRowCount) { - $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); - } - - if ($this->_invalidSoftCreditRowCount) { - $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } - } - - /** - * Export data to a CSV file. - * - * @param string $fileName - * @param array $header - * @param array $data - */ - public static function exportCSV($fileName, $header, $data) { - $output = []; - $fd = fopen($fileName, 'w'); - - foreach ($header as $key => $value) { - $header[$key] = "\"$value\""; - } - $config = CRM_Core_Config::singleton(); - $output[] = implode($config->fieldSeparator, $header); - - foreach ($data as $datum) { - foreach ($datum as $key => $value) { - if (isset($value[0]) && is_array($value)) { - foreach ($value[0] as $k1 => $v1) { - if ($k1 == 'location_type_id') { - continue; - } - $datum[$k1] = $v1; - } - } - else { - $datum[$key] = "\"$value\""; - } - } - $output[] = implode($config->fieldSeparator, $datum); - } - fwrite($fd, implode("\n", $output)); - fclose($fd); - } - - /** - * Determines the file extension based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function errorFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - $config = CRM_Core_Config::singleton(); - $fileName = $config->uploadDir . "sqlImport"; - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName .= '.softCreditErrors'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName .= '.pledgePaymentErrors'; - break; - - default: - $fileName = parent::errorFileName($type); - break; - } - - return $fileName; - } - - /** - * Determines the file name based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function saveFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName = 'Import_Soft_Credit_Errors.csv'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName = 'Import_Pledge_Payment_Errors.csv'; - break; - - default: - $fileName = parent::saveFileName($type); - break; - } - - return $fileName; - } - -} diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index a8a98ef50d..d5386b77e8 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -18,7 +18,7 @@ /** * Class to parse contribution csv files. */ -class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Parser { +class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { protected $_mapperKeys; @@ -51,6 +51,654 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa $this->_mapperSoftCreditType = &$mapperSoftCreditType; } + /** + * Contribution-specific result codes + * @see CRM_Import_Parser result code constants + */ + const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096; + + /** + * @var string + */ + protected $_fileName; + + /** + * Imported file size + * @var int + */ + protected $_fileSize; + + /** + * Separator being used + * @var string + */ + protected $_separator; + + /** + * Total number of lines in file + * @var int + */ + protected $_lineCount; + + /** + * Running total number of valid soft credit rows + * @var int + */ + protected $_validSoftCreditRowCount; + + /** + * Running total number of invalid soft credit rows + * @var int + */ + protected $_invalidSoftCreditRowCount; + + /** + * Running total number of valid pledge payment rows + * @var int + */ + protected $_validPledgePaymentRowCount; + + /** + * Running total number of invalid pledge payment rows + * @var int + */ + protected $_invalidPledgePaymentRowCount; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_pledgePaymentErrors; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_softCreditErrors; + + /** + * Filename of pledge payment error data + * + * @var string + */ + protected $_pledgePaymentErrorsFileName; + + /** + * Filename of soft credit error data + * + * @var string + */ + protected $_softCreditErrorsFileName; + + /** + * Whether the file has a column header or not + * + * @var bool + */ + protected $_haveColumnHeader; + + /** + * @param string $fileName + * @param string $separator + * @param $mapper + * @param bool $skipColumnHeader + * @param int $mode + * @param int $contactType + * @param int $onDuplicate + * @param int $statusID + * @param int $totalRowCount + * + * @return mixed + * @throws Exception + */ + public function run( + $fileName, + $separator, + &$mapper, + $skipColumnHeader = FALSE, + $mode = self::MODE_PREVIEW, + $contactType = self::CONTACT_INDIVIDUAL, + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL + ) { + if (!is_array($fileName)) { + throw new CRM_Core_Exception('Unable to determine import file'); + } + $fileName = $fileName['name']; + + switch ($contactType) { + case self::CONTACT_INDIVIDUAL: + $this->_contactType = 'Individual'; + break; + + case self::CONTACT_HOUSEHOLD: + $this->_contactType = 'Household'; + break; + + case self::CONTACT_ORGANIZATION: + $this->_contactType = 'Organization'; + } + + $this->init(); + + $this->_haveColumnHeader = $skipColumnHeader; + + $this->_separator = $separator; + + $fd = fopen($fileName, "r"); + if (!$fd) { + return FALSE; + } + + $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; + $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; + $this->_totalCount = $this->_conflictCount = 0; + + $this->_errors = []; + $this->_warnings = []; + $this->_conflicts = []; + $this->_pledgePaymentErrors = []; + $this->_softCreditErrors = []; + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } + + $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); + + if ($mode == self::MODE_MAPFIELD) { + $this->_rows = []; + } + else { + $this->_activeFieldCount = count($this->_activeFields); + } + + while (!feof($fd)) { + $this->_lineCount++; + + $values = fgetcsv($fd, 8192, $separator); + if (!$values) { + continue; + } + + self::encloseScrub($values); + + // skip column header if we're not in mapfield mode + if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { + $skipColumnHeader = FALSE; + continue; + } + + /* trim whitespace around the values */ + + $empty = TRUE; + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + + if (CRM_Utils_System::isNull($values)) { + continue; + } + + $this->_totalCount++; + + if ($mode == self::MODE_MAPFIELD) { + $returnCode = $this->mapField($values); + } + elseif ($mode == self::MODE_PREVIEW) { + $returnCode = $this->preview($values); + } + elseif ($mode == self::MODE_SUMMARY) { + $returnCode = $this->summary($values); + } + elseif ($mode == self::MODE_IMPORT) { + $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } + } + else { + $returnCode = self::ERROR; + } + + // note that a line could be valid but still produce a warning + if ($returnCode == self::VALID) { + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::SOFT_CREDIT) { + $this->_validSoftCreditRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::PLEDGE_PAYMENT) { + $this->_validPledgePaymentRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::WARNING) { + $this->_warningCount++; + if ($this->_warningCount < $this->_maxWarningCount) { + $this->_warningCount[] = $line; + } + } + + if ($returnCode == self::ERROR) { + $this->_invalidRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_errors[] = $values; + } + + if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { + $this->_invalidPledgePaymentRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_pledgePaymentErrors[] = $values; + } + + if ($returnCode == self::SOFT_CREDIT_ERROR) { + $this->_invalidSoftCreditRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_softCreditErrors[] = $values; + } + + if ($returnCode == self::CONFLICT) { + $this->_conflictCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_conflicts[] = $values; + } + + if ($returnCode == self::DUPLICATE) { + if ($returnCode == self::MULTIPLE_DUPE) { + /* TODO: multi-dupes should be counted apart from singles + * on non-skip action */ + } + $this->_duplicateCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_duplicates[] = $values; + if ($onDuplicate != self::DUPLICATE_SKIP) { + $this->_validCount++; + } + } + + // we give the derived class a way of aborting the process + // note that the return code could be multiple code or'ed together + if ($returnCode == self::STOP) { + break; + } + + // if we are done processing the maxNumber of lines, break + if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { + break; + } + } + + fclose($fd); + + if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { + $customHeaders = $mapper; + + $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); + foreach ($customHeaders as $key => $value) { + if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { + $customHeaders[$key] = $customfields[$id][0]; + } + } + if ($this->_invalidRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_errorFileName = self::errorFileName(self::ERROR); + self::exportCSV($this->_errorFileName, $headers, $this->_errors); + } + + if ($this->_invalidPledgePaymentRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); + self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); + } + + if ($this->_invalidSoftCreditRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); + self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); + } + + if ($this->_conflictCount) { + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_conflictFileName = self::errorFileName(self::CONFLICT); + self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); + } + if ($this->_duplicateCount) { + $headers = array_merge([ + ts('Line Number'), + ts('View Contribution URL'), + ], $customHeaders); + + $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); + self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); + } + } + return $this->fini(); + } + + /** + * Given a list of the importable field keys that the user has selected + * set the active fields array to this list + * + * @param array $fieldKeys mapped array of values + */ + public function setActiveFields($fieldKeys) { + $this->_activeFieldCount = count($fieldKeys); + foreach ($fieldKeys as $key) { + if (empty($this->_fields[$key])) { + $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); + } + else { + $this->_activeFields[] = clone($this->_fields[$key]); + } + } + } + + /** + * Store the soft credit field information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCredit($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditField = $element; + } + } + + /** + * Store the soft credit field type information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCreditType($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditType = $element; + } + } + + /** + * Format the field values for input to the api. + * + * @return array + * (reference ) associative array of name/value pairs + */ + public function &getActiveFieldParams() { + $params = []; + for ($i = 0; $i < $this->_activeFieldCount; $i++) { + if (isset($this->_activeFields[$i]->_value)) { + if (isset($this->_activeFields[$i]->_softCreditField)) { + if (!isset($params[$this->_activeFields[$i]->_name])) { + $params[$this->_activeFields[$i]->_name] = []; + } + $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; + if (isset($this->_activeFields[$i]->_softCreditType)) { + $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; + } + } + + if (!isset($params[$this->_activeFields[$i]->_name])) { + if (!isset($this->_activeFields[$i]->_softCreditField)) { + $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; + } + } + } + } + return $params; + } + + /** + * @param string $name + * @param $title + * @param int $type + * @param string $headerPattern + * @param string $dataPattern + */ + public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { + if (empty($name)) { + $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); + if (!array_key_exists($name, $tempField)) { + $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, + CRM_Utils_Array::value('hasLocationType', $tempField[$name]) + ); + } + } + } + + /** + * Store parser values. + * + * @param CRM_Core_Session $store + * + * @param int $mode + */ + public function set($store, $mode = self::MODE_SUMMARY) { + $store->set('fileSize', $this->_fileSize); + $store->set('lineCount', $this->_lineCount); + $store->set('separator', $this->_separator); + $store->set('fields', $this->getSelectValues()); + $store->set('fieldTypes', $this->getSelectTypes()); + + $store->set('headerPatterns', $this->getHeaderPatterns()); + $store->set('dataPatterns', $this->getDataPatterns()); + $store->set('columnCount', $this->_activeFieldCount); + + $store->set('totalRowCount', $this->_totalCount); + $store->set('validRowCount', $this->_validCount); + $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); + $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); + $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); + $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); + $store->set('conflictRowCount', $this->_conflictCount); + + switch ($this->_contactType) { + case 'Individual': + $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); + break; + + case 'Household': + $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); + break; + + case 'Organization': + $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); + } + + if ($this->_invalidRowCount) { + $store->set('errorsFileName', $this->_errorFileName); + } + if ($this->_conflictCount) { + $store->set('conflictsFileName', $this->_conflictFileName); + } + if (isset($this->_rows) && !empty($this->_rows)) { + $store->set('dataValues', $this->_rows); + } + + if ($this->_invalidPledgePaymentRowCount) { + $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); + } + + if ($this->_invalidSoftCreditRowCount) { + $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); + } + + if ($mode == self::MODE_IMPORT) { + $store->set('duplicateRowCount', $this->_duplicateCount); + if ($this->_duplicateCount) { + $store->set('duplicatesFileName', $this->_duplicateFileName); + } + } + } + + /** + * Export data to a CSV file. + * + * @param string $fileName + * @param array $header + * @param array $data + */ + public static function exportCSV($fileName, $header, $data) { + $output = []; + $fd = fopen($fileName, 'w'); + + foreach ($header as $key => $value) { + $header[$key] = "\"$value\""; + } + $config = CRM_Core_Config::singleton(); + $output[] = implode($config->fieldSeparator, $header); + + foreach ($data as $datum) { + foreach ($datum as $key => $value) { + if (isset($value[0]) && is_array($value)) { + foreach ($value[0] as $k1 => $v1) { + if ($k1 == 'location_type_id') { + continue; + } + $datum[$k1] = $v1; + } + } + else { + $datum[$key] = "\"$value\""; + } + } + $output[] = implode($config->fieldSeparator, $datum); + } + fwrite($fd, implode("\n", $output)); + fclose($fd); + } + + /** + * Determines the file extension based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function errorFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + $config = CRM_Core_Config::singleton(); + $fileName = $config->uploadDir . "sqlImport"; + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName .= '.softCreditErrors'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName .= '.pledgePaymentErrors'; + break; + + default: + $fileName = parent::errorFileName($type); + break; + } + + return $fileName; + } + + /** + * Determines the file name based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function saveFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName = 'Import_Soft_Credit_Errors.csv'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName = 'Import_Pledge_Payment_Errors.csv'; + break; + + default: + $fileName = parent::saveFileName($type); + break; + } + + return $fileName; + } + /** * The initializer code, called before the processing */ @@ -245,10 +893,10 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa if ($formatError) { array_unshift($values, $formatError['error_message']); if (CRM_Utils_Array::value('error_data', $formatError) == 'soft_credit') { - return CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR; + return self::SOFT_CREDIT_ERROR; } if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') { - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR; + return self::PLEDGE_PAYMENT_ERROR; } return CRM_Import_Parser::ERROR; } @@ -325,7 +973,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -379,7 +1027,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -446,7 +1094,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -477,7 +1125,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Pa $formatted['total_amount'] ); - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT; + return self::PLEDGE_PAYMENT; } } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index 2982049681..faac17af3f 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -70,7 +70,7 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase { ]; $mapperSoftCredit = [NULL, NULL, NULL, 'external_identifier']; $mapperSoftCreditType = [NULL, NULL, NULL, '1']; - $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); + $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); $contributionsOfMainContact = Contribution::get()->addWhere('contact_id', '=', $contact1Id)->execute(); $this->assertCount(1, $contributionsOfMainContact, 'Contribution not added for primary contact'); -- 2.25.1