From b584b608df17d49bddc1d11c5c7483ac01517ce7 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Sat, 4 Jun 2022 11:02:59 +1200 Subject: [PATCH] Member import cleanup - use datasource --- CRM/Member/Import/Form/MapField.php | 43 +++---------- CRM/Member/Import/Form/Preview.php | 36 ++--------- CRM/Member/Import/Parser/Membership.php | 82 +++---------------------- 3 files changed, 23 insertions(+), 138 deletions(-) diff --git a/CRM/Member/Import/Form/MapField.php b/CRM/Member/Import/Form/MapField.php index 08fc3605bd..4b2e302683 100644 --- a/CRM/Member/Import/Form/MapField.php +++ b/CRM/Member/Import/Form/MapField.php @@ -20,13 +20,6 @@ */ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { - /** - * store contactType. - * - * @var int - */ - public static $_contactType = NULL; - /** * Set variables up before form is built. * @@ -35,22 +28,15 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { public function preProcess() { $this->_mapperFields = $this->get('fields'); asort($this->_mapperFields); - - $this->_columnCount = $this->get('columnCount'); - $this->assign('columnCount', $this->_columnCount); - $this->_dataValues = $this->get('dataValues'); - $this->assign('dataValues', $this->_dataValues); + parent::preProcess(); $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); $this->_onDuplicate = $this->get('onDuplicate', $onDuplicate ?? ""); $highlightedFields = []; - if ($skipColumnHeader) { + if ($this->getSubmittedValue('skipColumnHeader')) { $this->assign('skipColumnHeader', $skipColumnHeader); $this->assign('rowDisplayCount', 3); - /* if we had a column header to skip, stash it for later */ - - $this->_columnHeaders = $this->_dataValues[0]; } else { $this->assign('rowDisplayCount', 2); @@ -58,7 +44,7 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { //CRM-2219 removing other required fields since for updation only //membership id is required. - if ($this->_onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { + if ($this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_UPDATE) { $remove = array('membership_contact_id', 'email', 'first_name', 'last_name', 'external_identifier'); foreach ($remove as $value) { unset($this->_mapperFields[$value]); @@ -85,8 +71,6 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { // modify field title $this->_mapperFields['status_id'] = ts('Membership Status'); $this->_mapperFields['membership_type_id'] = ts('Membership Type'); - - self::$_contactType = $this->get('contactType'); $this->assign('highlightedFields', $highlightedFields); } @@ -141,8 +125,8 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { $defaults = []; $mapperKeys = array_keys($this->_mapperFields); $hasHeaders = !empty($this->_columnHeaders); - $headerPatterns = $this->get('headerPatterns'); - $dataPatterns = $this->get('dataPatterns'); + $headerPatterns = $this->getHeaderPatterns(); + $dataPatterns = $this->getDataPatterns(); /* Initialize all field usages to false */ @@ -289,17 +273,10 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { 'membership_type_id' => ts('Membership Type'), 'membership_start_date' => ts('Membership Start Date'), ); - - $contactTypeId = $self->get('contactType'); - $contactTypes = array( - CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual', - CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household', - CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization', - ); - $params = array( - 'used' => 'Unsupervised', - 'contact_type' => $contactTypes[$contactTypeId], - ); + $params = array( + 'used' => 'Unsupervised', + 'contact_type' => $self->getContactType(), + ); list($ruleFields, $threshold) = CRM_Dedupe_BAO_DedupeRuleGroup::dedupeRuleFieldsWeight($params); $weightSum = 0; foreach ($importKeys as $key => $val) { @@ -415,7 +392,7 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { $parser = new CRM_Member_Import_Parser_Membership($mapperKeysMain); $parser->setUserJobID($this->getUserJobID()); $parser->run($this->getSubmittedValue('uploadFile'), $this->getSubmittedValue('fieldSeparator'), $mapper, $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_PREVIEW, $this->get('contactType') + CRM_Import_Parser::MODE_PREVIEW ); // add all the necessary variables to the form $parser->set($this); diff --git a/CRM/Member/Import/Form/Preview.php b/CRM/Member/Import/Form/Preview.php index 673dd472b0..718a0b4337 100644 --- a/CRM/Member/Import/Form/Preview.php +++ b/CRM/Member/Import/Form/Preview.php @@ -28,39 +28,12 @@ class CRM_Member_Import_Form_Preview extends CRM_Import_Form_Preview { */ public function preProcess() { parent::preProcess(); - //get the data from the session - $dataValues = $this->get('dataValues'); - $mapper = $this->get('mapper'); $invalidRowCount = $this->get('invalidRowCount'); - - //get the mapping name displayed if the mappingId is set - $mappingId = $this->get('loadMappingId'); - if ($mappingId) { - $mapDAO = new CRM_Core_DAO_Mapping(); - $mapDAO->id = $mappingId; - $mapDAO->find(TRUE); - } - $this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL); - if ($invalidRowCount) { $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Member_Import_Parser_Membership'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } - - $properties = [ - 'mapper', - 'dataValues', - 'columnCount', - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - ]; $this->setStatusUrl(); - - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } } /** @@ -89,12 +62,11 @@ class CRM_Member_Import_Form_Preview extends CRM_Import_Form_Preview { } $parser->run($this->getSubmittedValue('uploadFile'), $this->getSubmittedValue('fieldSeparator'), $mapperFields, - $this->getSubmittedValue('skipColumnHeader'), + NULL, CRM_Import_Parser::MODE_IMPORT, - $this->get('contactType'), - $onDuplicate, - $this->get('statusID'), - $this->get('totalRowCount') + NULL, + NULL, + $this->get('statusID') ); // add all the necessary variables to the form diff --git a/CRM/Member/Import/Parser/Membership.php b/CRM/Member/Import/Parser/Membership.php index 51abea6107..a000dc02b4 100644 --- a/CRM/Member/Import/Parser/Membership.php +++ b/CRM/Member/Import/Parser/Membership.php @@ -56,13 +56,6 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { */ protected $_lineCount; - /** - * Whether the file has a column header or not - * - * @var bool - */ - protected $_haveColumnHeader; - /** * Class constructor. * @@ -95,38 +88,11 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { $mode = self::MODE_PREVIEW, $contactType = self::CONTACT_INDIVIDUAL, $onDuplicate = self::DUPLICATE_SKIP, - $statusID = NULL, - $totalRowCount = NULL + $statusID = NULL ) { - if (!is_array($fileName)) { - throw new CRM_Core_Exception('Unable to determine import file'); - } - $fileName = $fileName['name']; - - switch ($contactType) { - case self::CONTACT_INDIVIDUAL: - $this->_contactType = 'Individual'; - break; - - case self::CONTACT_HOUSEHOLD: - $this->_contactType = 'Household'; - break; - - case self::CONTACT_ORGANIZATION: - $this->_contactType = 'Organization'; - } - + $this->_contactType = $this->getContactType(); $this->init(); - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - $this->_lineCount = 0; $this->_invalidRowCount = $this->_validCount = 0; $this->_totalCount = 0; @@ -134,8 +100,6 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { $this->_errors = []; $this->_warnings = []; - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - if ($mode == self::MODE_MAPFIELD) { $this->_rows = []; } @@ -146,32 +110,13 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { $this->progressImport($statusID); $startTimestamp = $currTimestamp = $prevTimestamp = CRM_Utils_Time::time(); } - - while (!feof($fd)) { + $dataSource = $this->getDataSourceObject(); + $totalRowCount = $dataSource->getRowCount(['new']); + $dataSource->setStatuses(['new']); + while ($row = $dataSource->getRow()) { + $values = array_values($row); $this->_lineCount++; - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - if (CRM_Utils_System::isNull($values)) { - continue; - } - $this->_totalCount++; if ($mode == self::MODE_MAPFIELD) { @@ -184,7 +129,7 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { $returnCode = $this->summary($values); } elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); + $returnCode = $this->import($this->getSubmittedValue('onDuplicate'), $values); if ($statusID && (($this->_lineCount % 50) == 0)) { $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); } @@ -214,19 +159,12 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { $recordNumber = $this->_lineCount; array_unshift($values, $recordNumber); $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { + if ($this->getSubmittedValue('onDuplicate') != self::DUPLICATE_SKIP) { $this->_validCount++; } } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } } - fclose($fd); - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { $customHeaders = $mapper; @@ -353,9 +291,7 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { * @return void */ public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); $store->set('dataPatterns', $this->getDataPatterns()); $store->set('columnCount', $this->_activeFieldCount); -- 2.25.1