From 7eebbdaa9a8ccca6b1abf42cbf4ebaf4f20a718a Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Tue, 7 Jun 2022 14:55:32 +1200 Subject: [PATCH] Fixes for Activity import --- CRM/Activity/Import/Form/MapField.php | 240 +++----------- CRM/Activity/Import/Form/Preview.php | 106 ------ CRM/Activity/Import/Form/Summary.php | 46 --- CRM/Activity/Import/Parser/Activity.php | 311 +++++------------- CRM/Event/Import/Form/MapField.php | 23 +- CRM/Import/Form/MapField.php | 21 ++ CRM/Import/Form/Summary.php | 18 + CRM/Member/Import/Form/MapField.php | 22 +- CRM/Upgrade/Incremental/php/FiveFiftyOne.php | 39 +++ .../CRM/Activity/Import/Form/MapField.tpl | 28 +- .../CRM/Activity/Import/Form/Preview.tpl | 2 +- templates/CRM/Import/Form/MapField.tpl | 36 ++ .../Activity/Import/Parser/ActivityTest.php | 195 ++++++++--- .../Activity/Import/Parser/data/activity.csv | 2 + .../Event/Import/Parser/ParticipantTest.php | 2 +- tests/phpunit/CiviTest/CiviUnitTestCase.php | 12 + 16 files changed, 409 insertions(+), 694 deletions(-) create mode 100644 templates/CRM/Import/Form/MapField.tpl create mode 100644 tests/phpunit/CRM/Activity/Import/Parser/data/activity.csv diff --git a/CRM/Activity/Import/Form/MapField.php b/CRM/Activity/Import/Form/MapField.php index e9b6c65106..92d9ffd39b 100644 --- a/CRM/Activity/Import/Form/MapField.php +++ b/CRM/Activity/Import/Form/MapField.php @@ -25,122 +25,39 @@ class CRM_Activity_Import_Form_MapField extends CRM_Import_Form_MapField { */ public $submitOnce = TRUE; - /** - * Set variables up before form is built. - */ - public function preProcess() { - $this->_mapperFields = $this->get('fields'); - unset($this->_mapperFields['id']); - asort($this->_mapperFields); - - $this->_columnCount = $this->get('columnCount'); - $this->assign('columnCount', $this->_columnCount); - $this->_dataValues = $this->get('dataValues'); - $this->assign('dataValues', $this->_dataValues); - - $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); - - if ($skipColumnHeader) { - $this->assign('skipColumnHeader', $skipColumnHeader); - $this->assign('rowDisplayCount', 3); - // If we had a column header to skip, stash it for later. - - $this->_columnHeaders = $this->_dataValues[0]; - } - else { - $this->assign('rowDisplayCount', 2); - } - $highlightedFields = []; - $requiredFields = [ - 'activity_date_time', - 'activity_type_id', - 'activity_label', - 'target_contact_id', - 'activity_subject', - ]; - foreach ($requiredFields as $val) { - $highlightedFields[] = $val; - } - $this->assign('highlightedFields', $highlightedFields); - } - /** * Build the form object. * * @throws \CiviCRM_API3_Exception */ public function buildQuickForm() { - // To save the current mappings. - if (!$this->get('savedMapping')) { - $saveDetailsName = ts('Save this field mapping'); - $this->applyFilter('saveMappingName', 'trim'); - $this->add('text', 'saveMappingName', ts('Name')); - $this->add('text', 'saveMappingDesc', ts('Description')); - } - else { - $savedMapping = $this->get('savedMapping'); - // Mapping is to be loaded from database. - - // Get an array of the name values for mapping fields associated with this mapping_id. - $mappingName = CRM_Core_BAO_Mapping::getMappingFieldValues($savedMapping, 'name'); - - $this->set('loadedMapping', $savedMapping); - - $params = ['id' => $savedMapping]; - $temp = []; - $mappingDetails = CRM_Core_BAO_Mapping::retrieve($params, $temp); - - $this->assign('savedMappingName', $mappingDetails->name); - - $this->add('hidden', 'mappingId', $savedMapping); - - $this->addElement('checkbox', 'updateMapping', ts('Update this field mapping'), NULL); - $saveDetailsName = ts('Save as a new field mapping'); - $this->add('text', 'saveMappingName', ts('Name')); - $this->add('text', 'saveMappingDesc', ts('Description')); - } - - $this->addElement('checkbox', 'saveMapping', $saveDetailsName, NULL, ['onclick' => "showSaveDetails(this)"]); - + $savedMappingID = (int) $this->getSubmittedValue('savedMapping'); + $this->buildSavedMappingFields($savedMappingID); $this->addFormRule(['CRM_Activity_Import_Form_MapField', 'formRule']); //-------- end of saved mapping stuff --------- $defaults = []; - $mapperKeys = array_keys($this->_mapperFields); - - $hasHeaders = !empty($this->_columnHeaders); - $headerPatterns = $this->get('headerPatterns'); - $dataPatterns = $this->get('dataPatterns'); - - // Initialize all field usages to false. + $headerPatterns = $this->getHeaderPatterns(); + $dataPatterns = $this->getDataPatterns(); + $fieldMappings = $this->getFieldMappings(); + $columnHeaders = $this->getColumnHeaders(); + $hasHeaders = $this->getSubmittedValue('skipColumnHeader'); - foreach ($mapperKeys as $key) { - $this->_fieldUsed[$key] = FALSE; - } $sel1 = $this->_mapperFields; $js = "\n"; $this->assign('initHideBoxes', $js); - // Set warning if mismatch in more than. - if (isset($mappingName)) { - if (($this->_columnCount != count($mappingName))) { - $warning++; - } - } - if ($warning != 0 && $this->get('savedMapping')) { - CRM_Core_Session::singleton()->setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); - } - else { - CRM_Core_Session::singleton()->setStatus(NULL); - } - $this->setDefaults($defaults); $this->addButtons([ @@ -311,91 +213,6 @@ class CRM_Activity_Import_Form_MapField extends CRM_Import_Form_MapField { return TRUE; } - /** - * Process the mapped fields and map it into the uploaded file. - * - * Preview the file and extract some summary statistics - */ - public function postProcess() { - $params = $this->controller->exportValues('MapField'); - // Reload the mapfield if load mapping is pressed. - if (!empty($params['savedMapping'])) { - $this->set('savedMapping', $params['savedMapping']); - $this->controller->resetPage($this->_name); - return; - } - $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); - - $mapper = []; - $mapperKeys = $this->controller->exportValue($this->_name, 'mapper'); - $mapperKeysMain = []; - - for ($i = 0; $i < $this->_columnCount; $i++) { - $mapper[$i] = $this->_mapperFields[$mapperKeys[$i][0]]; - $mapperKeysMain[$i] = $mapperKeys[$i][0]; - } - - $this->set('mapper', $mapper); - // store mapping Id to display it in the preview page - if (!empty($params['mappingId'])) { - $this->set('loadMappingId', $params['mappingId']); - } - - // Updating Mapping Records. - if (!empty($params['updateMapping'])) { - - $mappingFields = new CRM_Core_DAO_MappingField(); - $mappingFields->mapping_id = $params['mappingId']; - $mappingFields->find(); - - $mappingFieldsId = []; - while ($mappingFields->fetch()) { - if ($mappingFields->id) { - $mappingFieldsId[$mappingFields->column_number] = $mappingFields->id; - } - } - - for ($i = 0; $i < $this->_columnCount; $i++) { - $updateMappingFields = new CRM_Core_DAO_MappingField(); - $updateMappingFields->id = $mappingFieldsId[$i]; - $updateMappingFields->mapping_id = $params['mappingId']; - $updateMappingFields->column_number = $i; - - $updateMappingFields->name = $mapper[$i]; - $updateMappingFields->save(); - } - } - - // Saving Mapping Details and Records. - if (!empty($params['saveMapping'])) { - $mappingParams = [ - 'name' => $params['saveMappingName'], - 'description' => $params['saveMappingDesc'], - 'mapping_type_id' => CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Mapping', 'mapping_type_id', 'Import Activity'), - ]; - $saveMapping = CRM_Core_BAO_Mapping::add($mappingParams); - - for ($i = 0; $i < $this->_columnCount; $i++) { - $saveMappingFields = new CRM_Core_DAO_MappingField(); - $saveMappingFields->mapping_id = $saveMapping->id; - $saveMappingFields->column_number = $i; - - $saveMappingFields->name = $mapper[$i]; - $saveMappingFields->save(); - } - $this->set('savedMapping', $saveMappingFields->mapping_id); - } - - $parser = new CRM_Activity_Import_Parser_Activity($mapperKeysMain); - $parser->setUserJobID($this->getUserJobID()); - $parser->run($this->getSubmittedValue('uploadFile'), $this->getSubmittedValue('fieldSeparator'), $mapper, $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_PREVIEW - ); - - // add all the necessary variables to the form - $parser->set($this); - } - /** * @return CRM_Activity_Import_Parser_Activity */ @@ -408,4 +225,31 @@ class CRM_Activity_Import_Form_MapField extends CRM_Import_Form_MapField { return $this->parser; } + protected function getHighlightedFields(): array { + $highlightedFields = []; + $requiredFields = [ + 'activity_date_time', + 'activity_type_id', + 'target_contact_id', + 'activity_subject', + ]; + foreach ($requiredFields as $val) { + $highlightedFields[] = $val; + } + return $highlightedFields; + } + + public function getImportType(): string { + return 'Import Activity'; + } + + /** + * Get the mapping name per the civicrm_mapping_field.type_id option group. + * + * @return string + */ + public function getMappingTypeName(): string { + return 'Import Participant'; + } + } diff --git a/CRM/Activity/Import/Form/Preview.php b/CRM/Activity/Import/Form/Preview.php index 16f5087d4e..31c1397575 100644 --- a/CRM/Activity/Import/Form/Preview.php +++ b/CRM/Activity/Import/Form/Preview.php @@ -20,112 +20,6 @@ */ class CRM_Activity_Import_Form_Preview extends CRM_Import_Form_Preview { - /** - * Set variables up before form is built. - */ - public function preProcess() { - parent::preProcess(); - // Get the data from the session. - $dataValues = $this->get('dataValues'); - $mapper = $this->get('mapper'); - $invalidRowCount = $this->get('invalidRowCount'); - - // Get the mapping name displayed if the mappingId is set. - $mappingId = $this->get('loadMappingId'); - if ($mappingId) { - $mapDAO = new CRM_Core_DAO_Mapping(); - $mapDAO->id = $mappingId; - $mapDAO->find(TRUE); - } - $this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL); - - if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser_Activity'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - - $properties = [ - 'mapper', - 'dataValues', - 'columnCount', - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - ]; - $this->setStatusUrl(); - - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } - } - - /** - * Process the mapped fields and map it into the uploaded file. - * - * Preview the file and extract some summary statistics - */ - public function postProcess() { - $fileName = $this->controller->exportValue('DataSource', 'uploadFile'); - $invalidRowCount = $this->get('invalidRowCount'); - $onDuplicate = $this->get('onDuplicate'); - - $mapper = $this->controller->exportValue('MapField', 'mapper'); - $mapperKeys = []; - - foreach ($mapper as $key => $value) { - $mapperKeys[$key] = $mapper[$key][0]; - } - - $parser = new CRM_Activity_Import_Parser_Activity($mapperKeys); - $parser->setUserJobID($this->getUserJobID()); - $mapFields = $this->get('fields'); - - foreach ($mapper as $key => $value) { - $header = []; - if (isset($mapFields[$mapper[$key][0]])) { - $header[] = $mapFields[$mapper[$key][0]]; - } - $mapperFields[] = implode(' - ', $header); - } - $parser->run( - $this->getSubmittedValue('uploadFile'), - $this->getSubmittedValue('fieldSeparator'), - $mapperFields, - $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_IMPORT, - $onDuplicate, - $this->get('statusID'), - $this->get('totalRowCount') - ); - - // add all the necessary variables to the form - $parser->set($this, CRM_Import_Parser::MODE_IMPORT); - - // check if there is any error occurred - - $errorStack = CRM_Core_Error::singleton(); - $errors = $errorStack->getErrors(); - $errorMessage = []; - - if (is_array($errors)) { - foreach ($errors as $key => $value) { - $errorMessage[] = $value['message']; - } - - $errorFile = $fileName['name'] . '.error.log'; - - if ($fd = fopen($errorFile, 'w')) { - fwrite($fd, implode('\n', $errorMessage)); - } - fclose($fd); - - $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser_Activity'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - } - /** * @return CRM_Activity_Import_Parser_Activity */ diff --git a/CRM/Activity/Import/Form/Summary.php b/CRM/Activity/Import/Form/Summary.php index 1fd39b2f8a..0aab950b95 100644 --- a/CRM/Activity/Import/Form/Summary.php +++ b/CRM/Activity/Import/Form/Summary.php @@ -20,50 +20,4 @@ */ class CRM_Activity_Import_Form_Summary extends CRM_Import_Form_Summary { - /** - * Set variables up before form is built. - */ - public function preProcess() { - // set the error message path to display - $this->assign('errorFile', $this->get('errorFile')); - - $totalRowCount = $this->get('totalRowCount'); - $this->set('totalRowCount', $totalRowCount); - - $invalidRowCount = $this->get('invalidRowCount'); - $onDuplicate = $this->get('onDuplicate'); - - $this->assign('dupeError', FALSE); - - if ($onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { - $dupeActionString = ts('These records have been updated with the imported data.'); - } - elseif ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) { - $dupeActionString = ts('These records have been filled in with the imported data.'); - } - else { - // Skip by default. - - $dupeActionString = ts('These records have not been imported.'); - - $this->assign('dupeError', TRUE); - - // Only subtract dupes from successful import if we're skipping. - - $this->set('validRowCount', $totalRowCount - $invalidRowCount); - } - $this->assign('dupeActionString', $dupeActionString); - - $properties = [ - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - 'groupAdditions', - ]; - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } - } - } diff --git a/CRM/Activity/Import/Parser/Activity.php b/CRM/Activity/Import/Parser/Activity.php index d90ae708a2..cd186dbfb3 100644 --- a/CRM/Activity/Import/Parser/Activity.php +++ b/CRM/Activity/Import/Parser/Activity.php @@ -30,33 +30,12 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { */ protected $_newActivity; - protected $_fileName; - - /** - * Imported file size. - * @var int - */ - protected $_fileSize; - - /** - * Separator being used. - * @var string - */ - protected $_separator; - /** * Total number of lines in file. * @var int */ protected $_lineCount; - /** - * Whether the file has a column header or not. - * - * @var bool - */ - protected $_haveColumnHeader; - /** * Class constructor. * @@ -71,24 +50,9 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { * The initializer code, called before the processing. */ public function init() { - $activityContact = CRM_Activity_BAO_ActivityContact::import(); - $activityTarget['target_contact_id'] = $activityContact['contact_id']; - $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(), - $activityTarget - ); - - $fields = array_merge($fields, [ - 'source_contact_id' => [ - 'title' => ts('Source Contact'), - 'headerPattern' => '/Source.Contact?/i', - ], - 'activity_label' => [ - 'title' => ts('Activity Type Label'), - 'headerPattern' => '/(activity.)?type label?/i', - ], - ]); - - foreach ($fields as $name => $field) { + $this->setFieldMetadata(); + + foreach ($this->importableFieldsMetadata as $name => $field) { $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT); $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//'); $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//'); @@ -129,41 +93,12 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { * * @param array $values * The array of values belonging to this line. - * - * @return int - * CRM_Import_Parser::VALID for success or - * CRM_Import_Parser::ERROR for error. - * - * @throws \CRM_Core_Exception */ - public function import(&$values) { + public function import($values) { + $rowNumber = (int) ($values[array_key_last($values)]); // First make sure this is a valid line try { - $this->validateValues($values); - - $params = $this->getApiReadyParams($values); - // For date-Formats. - $session = CRM_Core_Session::singleton(); - $dateType = $session->get('dateTypes'); - - $customFields = CRM_Core_BAO_CustomField::getFields('Activity'); - - foreach ($params as $key => $val) { - if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { - if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') { - $this->formatCustomDate($params, $params, $dateType, $key); - } - elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') { - $params[$key] = CRM_Utils_String::strtoboolstr($val); - } - } - elseif ($key === 'activity_date_time') { - $params[$key] = CRM_Utils_Date::formatDate($val, $dateType); - } - elseif ($key === 'activity_subject') { - $params['subject'] = $val; - } - } + $params = $this->getMappedRow($values); if (empty($params['external_identifier']) && empty($params['target_contact_id'])) { @@ -175,20 +110,19 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { if (!empty($matchedIDs)) { if (count($matchedIDs) > 1) { - array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Multiple matching contact records detected for this row. The activity was not imported'); } $cid = $matchedIDs[0]; $params['target_contact_id'] = $cid; $params['version'] = 3; $newActivity = civicrm_api('activity', 'create', $params); if (!empty($newActivity['is_error'])) { - array_unshift($values, $newActivity['error_message']); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception($newActivity['error_message']); } $this->_newActivity[] = $newActivity['id']; - return CRM_Import_Parser::VALID; + $this->setImportStatus($rowNumber, 'IMPORTED', '', $newActivity['id']); + return; } // Using new Dedupe rule. @@ -220,8 +154,7 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { } } - array_unshift($values, 'No matching Contact found for (' . $disp . ')'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('No matching Contact found for (' . $disp . ')'); } if (!empty($params['external_identifier'])) { $targetContactId = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', @@ -231,30 +164,53 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { if (!empty($params['target_contact_id']) && $params['target_contact_id'] != $targetContactId ) { - array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']); } if ($targetContactId) { $params['target_contact_id'] = $targetContactId; } else { - array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('No Matching Contact for External ID:' . $params['external_identifier']); } } $params['version'] = 3; $newActivity = civicrm_api('activity', 'create', $params); if (!empty($newActivity['is_error'])) { - array_unshift($values, $newActivity['error_message']); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception($newActivity['error_message']); } } catch (CRM_Core_Exception $e) { - return $this->addError($values, [$e->getMessage()]); + $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage()); + return; } $this->_newActivity[] = $newActivity['id']; - return CRM_Import_Parser::VALID; + $this->setImportStatus($rowNumber, 'IMPORTED', '', $newActivity['id']); + } + + /** + * Get the row from the csv mapped to our parameters. + * + * @param array $values + * + * @return array + * @throws \API_Exception + */ + public function getMappedRow(array $values): array { + $params = []; + foreach ($this->getFieldMappings() as $i => $mappedField) { + if ($mappedField['name'] === 'do_not_import') { + continue; + } + if ($mappedField['name']) { + $fieldName = $this->getFieldMetadata($mappedField['name'])['name']; + if (in_array($mappedField['name'], ['target_contact_id', 'source_contact_id'])) { + $fieldName = $mappedField['name']; + } + $params[$fieldName] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); + } + } + return $params; } /** @@ -273,6 +229,13 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { return $row[$this->getFieldIndex($fieldName)] ?? NULL; } + /** + * @return array + */ + protected function getRequiredFields(): array { + return [['activity_type_id' => ts('Activity Type'), 'activity_date_time' => ts('Activity Date')]]; + } + /** * Get the index for the given field. * @@ -362,41 +325,6 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { } } - /** - * @param array $values - * - * @throws \CRM_Core_Exception - */ - public function validateValues(array $values): void { - // Check required fields if this is not an update. - if (!$this->getFieldValue($values, 'activity_id')) { - if (!$this->getFieldValue($values, 'activity_label') - && !$this->getFieldValue($values, 'activity_type_id')) { - throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID')); - } - if (!$this->getFieldValue($values, 'activity_date_time')) { - throw new CRM_Core_Exception(ts('Missing required fields')); - } - } - - $this->validateActivityTypeIDAndLabel($values); - if ($this->getFieldValue($values, 'activity_date_time') - && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) { - throw new CRM_Core_Exception(ts('Invalid Activity Date')); - } - - if ($this->getFieldValue($values, 'activity_engagement_level') - && !CRM_Utils_Rule::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) { - throw new CRM_Core_Exception(ts('Activity Engagement Index')); - } - - $targetContactID = $this->getFieldValue($values, 'target_contact_id'); - if ($targetContactID && !$this->isValidContactID($targetContactID)) { - throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type::escape($targetContactID, 'String')); - } - $this->validateCustomFields($values); - } - /** * Get array of parameters formatted for the api from the submitted values. * @@ -426,9 +354,6 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { * @param int $onDuplicate * @param int $statusID * @param int $totalRowCount - * - * @return mixed - * @throws Exception */ public function run( array $fileName, @@ -440,29 +365,14 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { $statusID = NULL, $totalRowCount = NULL ) { - - $fileName = $fileName['name']; - $this->init(); - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - $this->_lineCount = 0; $this->_invalidRowCount = $this->_validCount = 0; $this->_totalCount = 0; $this->_errors = []; $this->_warnings = []; - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - if ($mode == self::MODE_MAPFIELD) { $this->_rows = []; } @@ -474,31 +384,11 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { $startTimestamp = $currTimestamp = $prevTimestamp = time(); } - while (!feof($fd)) { + $dataSource = $this->getDataSourceObject(); + $dataSource->setStatuses(['new']); + while ($row = $dataSource->getRow()) { $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - // Trim whitespace around the values. - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - - if (CRM_Utils_System::isNull($values)) { - continue; - } - + $values = array_values($row); $this->_totalCount++; if ($mode == self::MODE_MAPFIELD) { @@ -509,60 +399,11 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { $returnCode = $this->summary($values); } elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($values); + $this->import($values); if ($statusID && (($this->_lineCount % 50) == 0)) { $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); } } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode & self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode & self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Activity'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge( - [ts('Line Number'), ts('Reason')], - $customHeaders - ); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } } } @@ -611,28 +452,7 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { * * @param CRM_Core_Session $store */ - public function set($store) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - } + public function set($store) {} /** * Export data to a CSV file. @@ -661,4 +481,25 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { fclose($fd); } + /** + * Ensure metadata is loaded. + */ + protected function setFieldMetadata(): void { + if (empty($this->importableFieldsMetadata)) { + $activityContact = CRM_Activity_BAO_ActivityContact::import(); + $activityTarget['target_contact_id'] = $activityContact['contact_id']; + $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(), + $activityTarget + ); + + $fields = array_merge($fields, [ + 'source_contact_id' => [ + 'title' => ts('Source Contact'), + 'headerPattern' => '/Source.Contact?/i', + ], + ]); + $this->importableFieldsMetadata = $fields; + } + } + } diff --git a/CRM/Event/Import/Form/MapField.php b/CRM/Event/Import/Form/MapField.php index 22c4741528..0ed3a16211 100644 --- a/CRM/Event/Import/Form/MapField.php +++ b/CRM/Event/Import/Form/MapField.php @@ -15,8 +15,6 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ -use Civi\Api4\MappingField; - /** * This class gets the name of the file to upload */ @@ -65,16 +63,7 @@ class CRM_Event_Import_Form_MapField extends CRM_Import_Form_MapField { $hasHeaders = $this->getSubmittedValue('skipColumnHeader'); $headerPatterns = $this->getHeaderPatterns(); $dataPatterns = $this->getDataPatterns(); - $savedMappingID = $this->getSubmittedValue('savedMapping'); - //used to warn for mismatch column count or mismatch mapping - $warning = 0; - if ($savedMappingID) { - $fieldMappings = MappingField::get(FALSE)->addWhere('mapping_id', '=', $savedMappingID)->execute()->indexBy('column_number'); - //set warning if mismatch in more than - if (($this->_columnCount != count($fieldMappings))) { - $warning++; - } - } + $fieldMappings = $this->getFieldMappings(); /* Initialize all field usages to false */ foreach ($mapperKeys as $key) { @@ -144,16 +133,6 @@ class CRM_Event_Import_Form_MapField extends CRM_Import_Form_MapField { } $js .= "\n"; $this->assign('initHideBoxes', $js); - - if ($warning != 0 && $this->getSubmittedValue('savedMapping')) { - $session = CRM_Core_Session::singleton(); - $session->setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); - } - else { - $session = CRM_Core_Session::singleton(); - $session->setStatus(NULL); - } - $this->setDefaults($defaults); $this->addButtons(array( diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index 8fb266dc4b..db3331ea9a 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -297,4 +297,25 @@ abstract class CRM_Import_Form_MapField extends CRM_Import_Forms { } } + /** + * @throws \API_Exception + * @throws \CRM_Core_Exception + * @throws \Civi\API\Exception\UnauthorizedException + */ + protected function getFieldMappings(): array { + $savedMappingID = $this->getSubmittedValue('savedMapping'); + if ($savedMappingID) { + $fieldMappings = MappingField::get(FALSE) + ->addWhere('mapping_id', '=', $savedMappingID) + ->execute() + ->indexBy('column_number'); + + if ((count($this->getColumnHeaders()) !== count($fieldMappings))) { + CRM_Core_Session::singleton()->setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); + } + return (array) $fieldMappings; + } + return []; + } + } diff --git a/CRM/Import/Form/Summary.php b/CRM/Import/Form/Summary.php index 72c8be2f27..461990ce01 100644 --- a/CRM/Import/Form/Summary.php +++ b/CRM/Import/Form/Summary.php @@ -15,6 +15,8 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\UserJob; + /** * This class summarizes the import results. * @@ -65,6 +67,22 @@ abstract class CRM_Import_Form_Summary extends CRM_Import_Forms { $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR)); $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH)); $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); + $userJobID = CRM_Utils_Request::retrieve('user_job_id', 'String', $this, TRUE); + $userJob = UserJob::get(TRUE)->addWhere('id', '=', $userJobID)->execute()->first(); + $onDuplicate = (int) $userJob['metadata']['submitted_values']['onDuplicate']; + $this->assign('dupeError', FALSE); + if ($onDuplicate === CRM_Import_Parser::DUPLICATE_UPDATE) { + $dupeActionString = ts('These records have been updated with the imported data.'); + } + elseif ($onDuplicate === CRM_Import_Parser::DUPLICATE_FILL) { + $dupeActionString = ts('These records have been filled in with the imported data.'); + } + else { + // Skip by default. + $dupeActionString = ts('These records have not been imported.'); + $this->assign('dupeError', TRUE); + } + $this->assign('dupeActionString', $dupeActionString); } } diff --git a/CRM/Member/Import/Form/MapField.php b/CRM/Member/Import/Form/MapField.php index ef105204e9..affa870170 100644 --- a/CRM/Member/Import/Form/MapField.php +++ b/CRM/Member/Import/Form/MapField.php @@ -50,12 +50,7 @@ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { $js = "\n"; $this->assign('initHideBoxes', $js); - //set warning if mismatch in more than - if (isset($mappingName)) { - if (($this->_columnCount != count($mappingName))) { - $warning++; - } - } - if ($warning != 0 && $this->get('savedMapping')) { - $session = CRM_Core_Session::singleton(); - $session::setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); - } - else { - $session = CRM_Core_Session::singleton(); - $session::setStatus(NULL); - } - $this->setDefaults($defaults); $this->addButtons(array( diff --git a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php index 1413b242b6..67f346acf5 100644 --- a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php @@ -159,6 +159,45 @@ class CRM_Upgrade_Incremental_php_FiveFiftyOne extends CRM_Upgrade_Incremental_B } } + // Activity fields... + // Yes - I know they could be combined - but it's also less confusing this way. + $mappings = MappingField::get(FALSE) + ->setSelect(['id', 'name']) + ->addWhere('mapping_id.mapping_type_id:name', '=', 'Import Activity') + ->execute(); + + $activityContact = CRM_Activity_BAO_ActivityContact::import(); + $activityTarget['target_contact_id'] = $activityContact['contact_id']; + $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(), + $activityTarget + ); + + $fields = array_merge($fields, [ + 'source_contact_id' => [ + 'title' => ts('Source Contact'), + 'headerPattern' => '/Source.Contact?/i', + ], + ]); + + $fieldMap = []; + foreach ($fields as $fieldName => $field) { + $fieldMap[$field['title']] = $fieldName; + if (!empty($field['html']['label'])) { + $fieldMap[$field['html']['label']] = $fieldName; + } + } + $fieldMap[ts('- do not import -')] = 'do_not_import'; + $fieldMap[ts('Activity Type Label')] = 'activity_type_id'; + + foreach ($mappings as $mapping) { + if (!empty($fieldMap[$mapping['name']])) { + MappingField::update(FALSE) + ->addWhere('id', '=', $mapping['id']) + ->addValue('name', $fieldMap[$mapping['name']]) + ->execute(); + } + } + return TRUE; } diff --git a/templates/CRM/Activity/Import/Form/MapField.tpl b/templates/CRM/Activity/Import/Form/MapField.tpl index dc6576a003..8564c9b70f 100644 --- a/templates/CRM/Activity/Import/Form/MapField.tpl +++ b/templates/CRM/Activity/Import/Form/MapField.tpl @@ -7,30 +7,4 @@ | and copyright information, see https://civicrm.org/licensing | +--------------------------------------------------------------------+ *} -
-{* Activity Import Wizard - Step 2 (map incoming data fields) *} -{* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *} - - {* WizardHeader.tpl provides visual display of steps thru the wizard as well as title for current step *} - {include file="CRM/common/WizardHeader.tpl"} - -
-

{ts}Review the values shown below from the first 2 rows of your import file and select the matching CiviCRM database fields from the drop-down lists in the right-hand column. Select '- do not import -' for any columns in the import file that you want ignored.{/ts}

-

{ts}If you think you may be importing additional data from the same data source, check 'Save this field mapping' at the bottom of the page before continuing. The saved mapping can then be easily reused the next time data is imported.{/ts}

-
-
{include file="CRM/common/formButtons.tpl" location="top"}
- {* Table for mapping data to CRM fields *} - {include file="CRM/Import/Form/MapTable.tpl"} -
- -
{include file="CRM/common/formButtons.tpl" location="bottom"}
- {$initHideBoxes|smarty:nodefaults} -{literal} - -{/literal} -
+{include file="CRM/Import/Form/MapField.tpl"} diff --git a/templates/CRM/Activity/Import/Form/Preview.tpl b/templates/CRM/Activity/Import/Form/Preview.tpl index 33b4be0009..d9b0e7392c 100644 --- a/templates/CRM/Activity/Import/Form/Preview.tpl +++ b/templates/CRM/Activity/Import/Form/Preview.tpl @@ -55,7 +55,7 @@
{* Table for mapping preview *} - {include file="CRM/Import/Form/MapTable.tpl"} + {include file="CRM/Import/Form/MapTableCommon.tpl"}
{include file="CRM/common/formButtons.tpl" location="bottom"}
diff --git a/templates/CRM/Import/Form/MapField.tpl b/templates/CRM/Import/Form/MapField.tpl new file mode 100644 index 0000000000..7a543b6185 --- /dev/null +++ b/templates/CRM/Import/Form/MapField.tpl @@ -0,0 +1,36 @@ +{* + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC. All rights reserved. | + | | + | This work is published under the GNU AGPLv3 license with some | + | permitted exceptions and without any warranty. For full license | + | and copyright information, see https://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} +
+ {* Import Wizard - Step 2 (map incoming data fields) *} + {* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *} + + {* WizardHeader.tpl provides visual display of steps thru the wizard as well as title for current step *} + {include file="CRM/common/WizardHeader.tpl"} + +
+

{ts}Review the values shown below from the first 2 rows of your import file and select the matching CiviCRM database fields from the drop-down lists in the right-hand column. Select '- do not import -' for any columns in the import file that you want ignored.{/ts}

+

{ts}If you think you may be importing additional data from the same data source, check 'Save this field mapping' at the bottom of the page before continuing. The saved mapping can then be easily reused the next time data is imported.{/ts}

+
+
{include file="CRM/common/formButtons.tpl" location="top"}
+ {* Table for mapping data to CRM fields *} + {include file="CRM/Import/Form/MapTableCommon.tpl" mapper=$form.mapper} +
+ +
{include file="CRM/common/formButtons.tpl" location="bottom"}
+ {$initHideBoxes|smarty:nodefaults} + {literal} + + {/literal} +
diff --git a/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php b/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php index 7205b16dc3..e3ce616250 100644 --- a/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php +++ b/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php @@ -18,6 +18,8 @@ * . */ +use Civi\Api4\UserJob; + /** * Test Activity Import Parser functions * @@ -27,6 +29,11 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { use CRMTraits_Custom_CustomDataTrait; + /** + * @var int|null + */ + private $userJobID; + /** * Prepare for tests. */ @@ -41,7 +48,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { * @throws \CRM_Core_Exception */ public function tearDown():void { - $this->quickCleanup(['civicrm_contact', 'civicrm_activity', 'civicrm_activity_contact'], TRUE); + $this->quickCleanup(['civicrm_contact', 'civicrm_email', 'civicrm_activity', 'civicrm_activity_contact', 'civicrm_user_job', 'civicrm_queue', 'civicrm_queue_item'], TRUE); parent::tearDown(); } @@ -73,15 +80,13 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { * @return \CRM_Activity_Import_Parser_Activity */ protected function createImportObject(array $fields): \CRM_Activity_Import_Parser_Activity { - // @todo Eyes are weary so sanity-check this later: - // This loop seems the same as array_values($fields)? And this appears - // to only be called from one place that already has them sequentially - // indexed so is it even needed? - $fieldMapper = []; - foreach ($fields as $index => $field) { - $fieldMapper[] = $field; + $mapper = []; + foreach ($fields as $field) { + $mapper[] = [$field]; } - $importer = new CRM_Activity_Import_Parser_Activity($fieldMapper); + $importer = new CRM_Activity_Import_Parser_Activity(); + $this->userJobID = $this->getUserJobID(['mapper' => $mapper]); + $importer->setUserJobID($this->userJobID); $importer->init(); return $importer; } @@ -95,12 +100,30 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { */ protected function importValues(array $values, $expectedOutcome = 1): string { $importer = $this->createImportObject(array_keys($values)); + try { + $importer->validateValues(array_values($values)); + } + catch (CRM_Core_Exception $e) { + if ($expectedOutcome === 4) { + return $e->getMessage(); + } + throw $e; + } + // Stand in for rowNumber. + $values[] = 1; $params = array_values($values); - CRM_Core_Session::singleton()->set('dateTypes', 1); - $outcome = $importer->import($params); - $this->assertEquals($expectedOutcome, $outcome); - // If there was an error it's in element 0 - return $outcome === CRM_Import_Parser::VALID ? '' : $params[0]; + $importer->import($params); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + + $row = $dataSource->getRow(); + if ($expectedOutcome === 1) { + $this->assertEquals('IMPORTED', $row['_status']); + return CRM_Import_Parser::VALID; + } + if ($expectedOutcome === 4) { + $this->assertEquals('ERROR', $row['_status']); + return $row['_status_message']; + } } /** @@ -168,7 +191,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 2 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], @@ -178,27 +201,15 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 3 => [ 'input' => [ 'activity_type_id' => 1, - 'activity_label' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], 'expected_error' => '', ], - 4 => [ - 'input' => [ - 'activity_type_id' => 2, - 'activity_label' => 'Meeting', - 'activity_date_time' => $some_date, - 'activity_subject' => 'asubj', - ], - 'expected_error' => 'Activity type label and Activity type ID are in conflict', - ], - 5 => [ 'input' => [ 'activity_type_id' => 1, - 'activity_label' => '', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], @@ -207,8 +218,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 6 => [ 'input' => [ - 'activity_type_id' => '', - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], @@ -226,7 +236,6 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 8 => [ 'input' => [ 'activity_type_id' => '', - 'activity_label' => '', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], @@ -235,7 +244,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 9 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_subject' => 'asubj', ], 'expected_error' => 'Missing required fields', @@ -243,7 +252,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 10 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => '', 'activity_subject' => 'asubj', ], @@ -255,7 +264,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { // is correct and it shouldn't be required in UI. 11 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, ], 'expected_error' => '', @@ -266,7 +275,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { // is correct and it shouldn't be required in UI. 12 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => '', ], @@ -275,7 +284,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 13 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', 'replace_me_custom_field' => 'InvalidValue', @@ -285,7 +294,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 14 => [ 'input' => [ - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', 'replace_me_custom_field' => '', @@ -296,7 +305,7 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { 15 => [ 'input' => [ 'target_contact_id' => '', - 'activity_label' => 'Meeting', + 'activity_type_id' => 'Meeting', 'activity_date_time' => $some_date, 'activity_subject' => 'asubj', ], @@ -306,4 +315,116 @@ class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase { ]; } + /** + * Import the csv file values. + * + * This function uses a flow that mimics the UI flow. + * + * @param string $csv Name of csv file. + * @param array $fieldMappings + * @param array $submittedValues + */ + protected function importCSV(string $csv, array $fieldMappings, array $submittedValues = []): void { + $submittedValues = array_merge([ + 'uploadFile' => ['name' => __DIR__ . '/data/' . $csv], + 'skipColumnHeader' => TRUE, + 'fieldSeparator' => ',', + 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'mapper' => $this->getMapperFromFieldMappings($fieldMappings), + 'dataSource' => 'CRM_Import_DataSource_CSV', + 'file' => ['name' => $csv], + 'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd, + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, + 'groups' => [], + ], $submittedValues); + /* @var \CRM_Activity_Import_Form_DataSource $form */ + $form = $this->getFormObject('CRM_Activity_Import_Form_DataSource', $submittedValues); + $values = $_SESSION['_' . $form->controller->_name . '_container']['values']; + $form->buildForm(); + $form->postProcess(); + // This gets reset in DataSource so re-do.... + $_SESSION['_' . $form->controller->_name . '_container']['values'] = $values; + + $this->userJobID = $form->getUserJobID(); + /* @var CRM_Activity_Import_Form_MapField $form */ + $form = $this->getFormObject('CRM_Activity_Import_Form_MapField', $submittedValues); + $form->setUserJobID($this->userJobID); + $form->buildForm(); + $form->postProcess(); + /* @var CRM_Activity_Import_Form_Preview $form */ + $form = $this->getFormObject('CRM_Activity_Import_Form_Preview', $submittedValues); + $form->setUserJobID($this->userJobID); + $form->buildForm(); + $form->postProcess(); + } + + /** + * @param array $mappings + * + * @return array + */ + protected function getMapperFromFieldMappings(array $mappings): array { + $mapper = []; + foreach ($mappings as $mapping) { + $fieldInput = [$mapping['name']]; + $mapper[] = $fieldInput; + } + return $mapper; + } + + /** + * Test the full form-flow import. + */ + public function testImportCSV() :void { + $this->individualCreate(['email' => 'mum@example.com']); + $this->importCSV('activity.csv', [ + ['name' => 'activity_date_time'], + ['name' => 'activity_status_id'], + ['name' => 'email'], + ['name' => 'activity_type_id'], + ['name' => 'activity_details'], + ['name' => 'activity_duration'], + ['name' => 'priority_id'], + ['name' => 'activity_location'], + ['name' => 'activity_subject'], + ['name' => 'do_not_import'], + ]); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $row = $dataSource->getRow(); + $this->assertEquals('IMPORTED', $row['_status']); + $this->callAPISuccessGetSingle('Activity', ['priority_id' => 'Urgent']); + } + + /** + * @param array $submittedValues + * + * @return int + * @noinspection PhpDocMissingThrowsInspection + */ + protected function getUserJobID(array $submittedValues = []): int { + $userJobID = UserJob::create()->setValues([ + 'metadata' => [ + 'submitted_values' => array_merge([ + 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'contactSubType' => '', + 'dataSource' => 'CRM_Import_DataSource_SQL', + 'sqlQuery' => 'SELECT first_name FROM civicrm_contact', + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP, + 'dedupe_rule_id' => NULL, + 'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd, + ], $submittedValues), + ], + 'status_id:name' => 'draft', + 'type_id:name' => 'activity_import', + ])->execute()->first()['id']; + if ($submittedValues['dataSource'] ?? NULL === 'CRM_Import_DataSource') { + $dataSource = new CRM_Import_DataSource_CSV($userJobID); + } + else { + $dataSource = new CRM_Import_DataSource_SQL($userJobID); + } + $dataSource->initialize(); + return $userJobID; + } + } diff --git a/tests/phpunit/CRM/Activity/Import/Parser/data/activity.csv b/tests/phpunit/CRM/Activity/Import/Parser/data/activity.csv new file mode 100644 index 0000000000..5f5ec9b381 --- /dev/null +++ b/tests/phpunit/CRM/Activity/Import/Parser/data/activity.csv @@ -0,0 +1,2 @@ +Activity Date,Activity Status,Email (match to contact),Activity Type,Details,Duration,Priority,Location,Subject,Blah +2022-12-07 07:55:56,Completed,mum@example.com,Email,Some stuff,77,Urgent,Phoned up,Long chat,ignore diff --git a/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php b/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php index 1d6f1ba6c2..ab521a555b 100644 --- a/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php +++ b/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php @@ -1,7 +1,7 @@ controller->_name . '_container']['values']['Preview'] = $formValues; return $form; + case 'CRM_Activity_Import_Form_DataSource': + case 'CRM_Activity_Import_Form_MapField': + case 'CRM_Activity_Import_Form_Preview': + $form->controller = new CRM_Activity_Import_Controller(); + $form->controller->setStateMachine(new CRM_Core_StateMachine($form->controller)); + // The submitted values should be set on one or the other of the forms in the flow. + // For test simplicity we set on all rather than figuring out which ones go where.... + $_SESSION['_' . $form->controller->_name . '_container']['values']['DataSource'] = $formValues; + $_SESSION['_' . $form->controller->_name . '_container']['values']['MapField'] = $formValues; + $_SESSION['_' . $form->controller->_name . '_container']['values']['Preview'] = $formValues; + return $form; + case strpos($class, '_Form_') !== FALSE: $form->controller = new CRM_Core_Controller_Simple($class, $pageName); break; -- 2.25.1