From 9ff5f6c020705d087edd36789ae9dda5df0eba18 Mon Sep 17 00:00:00 2001 From: Nileema Date: Tue, 14 Jan 2014 16:59:52 +0530 Subject: [PATCH] -- HR-235 Import multi-value custom data --- CRM/Core/BAO/CustomGroup.php | 11 + CRM/Core/xml/Menu/Import.xml | 9 + CRM/Custom/Import/Controller.php | 23 ++ CRM/Custom/Import/Field.php | 28 ++ CRM/Custom/Import/Form/DataSource.php | 215 ++++++++++ CRM/Custom/Import/Form/MapField.php | 223 +++++++++++ CRM/Custom/Import/Form/Preview.php | 145 +++++++ CRM/Custom/Import/Form/Summary.php | 14 + CRM/Custom/Import/Parser.php | 373 ++++++++++++++++++ CRM/Custom/Import/Parser/Api.php | 247 ++++++++++++ CRM/Utils/Type.php | 20 +- .../CRM/Custom/Import/Form/DataSource.hlp | 33 ++ .../CRM/Custom/Import/Form/DataSource.tpl | 94 +++++ templates/CRM/Custom/Import/Form/MapField.tpl | 26 ++ templates/CRM/Custom/Import/Form/Preview.tpl | 26 ++ templates/CRM/Custom/Import/Form/Summary.tpl | 26 ++ 16 files changed, 1512 insertions(+), 1 deletion(-) create mode 100644 CRM/Custom/Import/Controller.php create mode 100644 CRM/Custom/Import/Field.php create mode 100644 CRM/Custom/Import/Form/DataSource.php create mode 100644 CRM/Custom/Import/Form/MapField.php create mode 100644 CRM/Custom/Import/Form/Preview.php create mode 100644 CRM/Custom/Import/Form/Summary.php create mode 100644 CRM/Custom/Import/Parser.php create mode 100644 CRM/Custom/Import/Parser/Api.php create mode 100644 templates/CRM/Custom/Import/Form/DataSource.hlp create mode 100644 templates/CRM/Custom/Import/Form/DataSource.tpl create mode 100644 templates/CRM/Custom/Import/Form/MapField.tpl create mode 100644 templates/CRM/Custom/Import/Form/Preview.tpl create mode 100644 templates/CRM/Custom/Import/Form/Summary.tpl diff --git a/CRM/Core/BAO/CustomGroup.php b/CRM/Core/BAO/CustomGroup.php index 775e260391..0b4aa7ae41 100644 --- a/CRM/Core/BAO/CustomGroup.php +++ b/CRM/Core/BAO/CustomGroup.php @@ -2212,5 +2212,16 @@ SELECT civicrm_custom_group.id as groupID, civicrm_custom_group.title as groupT return $hasReachedMax; } + static function getMultipleFieldGroup() { + $multipleGroup = array(); + $dao = new CRM_Core_DAO_CustomGroup(); + $dao->is_multiple = 1 ; + $dao->find(); + while($dao->fetch()) { + $multipleGroup[$dao->id] = $dao->title; + } + return $multipleGroup; + } + } diff --git a/CRM/Core/xml/Menu/Import.xml b/CRM/Core/xml/Menu/Import.xml index 13e478d7f1..e874e2a5b0 100644 --- a/CRM/Core/xml/Menu/Import.xml +++ b/CRM/Core/xml/Menu/Import.xml @@ -25,6 +25,15 @@ CRM_Activity_Import_Controller 420 + + civicrm/import/custom + id=%%id%% + Import Multi-value Custom Data + access CiviCRM + 1 + CRM_Custom_Import_Controller + 420 + civicrm/ajax/status CRM_Contact_Import_Page_AJAX::status diff --git a/CRM/Custom/Import/Controller.php b/CRM/Custom/Import/Controller.php new file mode 100644 index 0000000000..7720c428e8 --- /dev/null +++ b/CRM/Custom/Import/Controller.php @@ -0,0 +1,23 @@ +_stateMachine = new CRM_Import_StateMachine($this, $action); + + // create and instantiate the pages + $this->addPages($this->_stateMachine, $action); + + // add all the actions + $config = CRM_Core_Config::singleton(); + $this->addActions($config->uploadDir, array('uploadFile')); + } +} \ No newline at end of file diff --git a/CRM/Custom/Import/Field.php b/CRM/Custom/Import/Field.php new file mode 100644 index 0000000000..78e6e0a81b --- /dev/null +++ b/CRM/Custom/Import/Field.php @@ -0,0 +1,28 @@ +_id = CRM_Utils_Request::retrieve('id', 'Positive', $this, FALSE); + if ($this->_id) { + $params = "reset=1&id={$this->_id}"; + } + else { + $params = "reset=1"; + } + $session->pushUserContext(CRM_Utils_System::url('civicrm/import/custom', $params)); + } + + function setDefaultValues() { + $config = CRM_Core_Config::singleton(); + $defaults = array( + 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'fieldSeparator' => $config->fieldSeparator, + 'multipleCustomData' => $this->_id, + ); + + if ($loadeMapping = $this->get('loadedMapping')) { + $this->assign('loadedMapping', $loadeMapping); + $defaults['savedMapping'] = $loadeMapping; + } + + return $defaults; + } + + /** + * Function to actually build the form + * + * @return void + * @access public + */ + public function buildQuickForm() { + $multipleCustomData = CRM_Core_BAO_CustomGroup::getMultipleFieldGroup(); + $this->add('select', 'multipleCustomData', ts('Multi-value Custom Data'), array('' => ts('- select -')) + $multipleCustomData, TRUE); + + //Setting Upload File Size + $config = CRM_Core_Config::singleton(); + if ($config->maxImportFileSize >= 8388608) { + $uploadFileSize = 8388608; + } + else { + $uploadFileSize = $config->maxImportFileSize; + } + $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); + + $this->assign('uploadSize', $uploadSize); + $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); + + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->setMaxFileSize($uploadFileSize); + $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); + + $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); + + //get the saved mapping details + $mappingArray = CRM_Core_BAO_Mapping::getMappings(CRM_Core_OptionGroup::getValue('mapping_type', + 'Import Multi value custom data', + 'name' + )); + $this->assign('savedMapping', $mappingArray); + $this->add('select', 'savedMapping', ts('Mapping Option'), array('' => ts('- select -')) + $mappingArray); + + if ($loadeMapping = $this->get('loadedMapping')) { + $this->assign('loadedMapping', $loadeMapping); + $this->setDefaults(array('savedMapping' => $loadeMapping)); + } + + //contact types option + $contactOptions = array(); + if (CRM_Contact_BAO_ContactType::isActive('Individual')) { + $contactOptions[] = $this->createElement('radio', + NULL, NULL, ts('Individual'), CRM_Import_Parser::CONTACT_INDIVIDUAL + ); + } + if (CRM_Contact_BAO_ContactType::isActive('Household')) { + $contactOptions[] = $this->createElement('radio', + NULL, NULL, ts('Household'), CRM_Import_Parser::CONTACT_HOUSEHOLD + ); + } + if (CRM_Contact_BAO_ContactType::isActive('Organization')) { + $contactOptions[] = $this->createElement('radio', + NULL, NULL, ts('Organization'), CRM_Import_Parser::CONTACT_ORGANIZATION + ); + } + + $this->addGroup($contactOptions, 'contactType', + ts('Contact Type') + ); + + $this->setDefaults(array( + 'contactType' => + CRM_Import_Parser::CONTACT_INDIVIDUAL, + )); + + //build date formats + CRM_Core_Form_Date::buildAllowedDateFormats($this); + + $this->addButtons(array( + array( + 'type' => 'upload', + 'name' => ts('Continue >>'), + 'spacing' => '          ', + 'isDefault' => TRUE, + ), + array( + 'type' => 'cancel', + 'name' => ts('Cancel'), + ), + ) + ); + } + + /** + * Process the uploaded file + * + * @return void + * @access public + */ + public function postProcess() { + $this->controller->resetPage('MapField'); + + $fileName = $this->controller->exportValue($this->_name, 'uploadFile'); + $skipColumnHeader = $this->controller->exportValue($this->_name, 'skipColumnHeader'); + $contactType = $this->controller->exportValue($this->_name, 'contactType'); + $dateFormats = $this->controller->exportValue($this->_name, 'dateFormats'); + $savedMapping = $this->controller->exportValue($this->_name, 'savedMapping'); + $multipleCustomData = $this->controller->exportValue($this->_name, 'multipleCustomData'); + + $this->set('contactType', $contactType); + $this->set('dateFormats', $dateFormats); + $this->set('savedMapping', $savedMapping); + $this->set('multipleCustomData', $multipleCustomData); + + $session = CRM_Core_Session::singleton(); + $session->set("dateTypes", $dateFormats); + + $config = CRM_Core_Config::singleton(); + $seperator = $config->fieldSeparator; + + $mapper = array(); + + $parser = new CRM_Custom_Import_Parser_Api($mapper); + $parser->setEntity($multipleCustomData); + + $parser->setMaxLinesToProcess(100); + $parser->run($fileName, $seperator, + $mapper, + $skipColumnHeader, + CRM_Import_Parser::MODE_MAPFIELD, $contactType + ); + + // add all the necessary variables to the form + $parser->set($this); + } + + /** + * Return a descriptive name for the page, used in wizard header + * + * @return string + * @access public + */ + public function getTitle() { + return ts('Upload Data'); + } +} \ No newline at end of file diff --git a/CRM/Custom/Import/Form/MapField.php b/CRM/Custom/Import/Form/MapField.php new file mode 100644 index 0000000000..de34d89e43 --- /dev/null +++ b/CRM/Custom/Import/Form/MapField.php @@ -0,0 +1,223 @@ +_mapperFields = $this->get('fields'); + asort($this->_mapperFields); + $this->_columnCount = $this->get('columnCount'); + $this->_columnCount = $this->get('columnCount'); + $this->assign('columnCount', $this->_columnCount); + $this->_dataValues = $this->get('dataValues'); + $this->assign('dataValues', $this->_dataValues); + $this->_entity = $this->_multipleCustomData = $this->get('multipleCustomData'); + + $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); + $this->_onDuplicate = $this->get('onDuplicate'); + if ($skipColumnHeader) { + $this->assign('skipColumnHeader', $skipColumnHeader); + $this->assign('rowDisplayCount', 3); + /* if we had a column header to skip, stash it for later */ + + $this->_columnHeaders = $this->_dataValues[0]; + } + else { + $this->assign('rowDisplayCount', 2); + } + $this->assign('highlightedFields', $this->_highlightedFields); + } + + /** + * Function to actually build the form + * + * @return void + * @access public + */ + public function buildQuickForm() { + parent::buildQuickForm(); + $this->addFormRule(array('CRM_Custom_Import_Form_MapField', 'formRule')); + } + + /** + * global validation rules for the form + * + * @param array $fields posted values of the form + * + * @return array list of errors to be posted back to the form + * @static + * @access public + */ + static function formRule($fields, $files, $self) { + $errors = array(); + $fieldMessage = NULL; + if (!array_key_exists('savedMapping', $fields)) { + $importKeys = array(); + foreach ($fields['mapper'] as $mapperPart) { + $importKeys[] = $mapperPart[0]; + } + $requiredFields = array( + 'contact_id' => ts('Contact ID'), + ); + foreach ($requiredFields as $field => $title) { + if (!in_array($field, $importKeys)) { + if (!isset($errors['_qf_default'])) { + $errors['_qf_default'] = ''; + } + $errors['_qf_default'] .= ts('Missing required field: %1', array(1 => $title)); + } + } + } + + if (CRM_Utils_Array::value('saveMapping', $fields)) { + $nameField = CRM_Utils_Array::value('saveMappingName', $fields); + if (empty($nameField)) { + $errors['saveMappingName'] = ts('Name is required to save Import Mapping'); + } + else { + $mappingTypeId = CRM_Core_OptionGroup::getValue('mapping_type', $self->_mappingType, 'name'); + if (CRM_Core_BAO_Mapping::checkMapping($nameField, $mappingTypeId)) { + $errors['saveMappingName'] = ts('Duplicate ' . $self->_mappingType . 'Mapping Name'); + } + } + } + + //display Error if loaded mapping is not selected + if (array_key_exists('loadMapping', $fields)) { + $getMapName = CRM_Utils_Array::value('savedMapping', $fields); + if (empty($getMapName)) { + $errors['savedMapping'] = ts('Select saved mapping'); + } + } + + if (!empty($errors)) { + if (!empty($errors['saveMappingName'])) { + $_flag = 1; + $assignError = new CRM_Core_Page(); + $assignError->assign('mappingDetailsError', $_flag); + } + return $errors; + } + + return TRUE; + } + + /** + * Process the mapped fields and map it into the uploaded file + * preview the file and extract some summary statistics + * + * @return void + * @access public + */ + public function postProcess() { + $params = $this->controller->exportValues('MapField'); + $this->set('multipleCustomData', $this->_multipleCustomData); + + //reload the mapfield if load mapping is pressed + if (!empty($params['savedMapping'])) { + $this->set('savedMapping', $params['savedMapping']); + $this->controller->resetPage($this->_name); + return; + } + + $fileName = $this->controller->exportValue('DataSource', 'uploadFile'); + $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); + $this->_entity = $this->controller->exportValue('DataSource', 'entity'); + + $config = CRM_Core_Config::singleton(); + $separator = $config->fieldSeparator; + + $mapperKeys = array(); + $mapper = array(); + $mapperKeys = $this->controller->exportValue($this->_name, 'mapper'); + $mapperKeysMain = array(); + + for ($i = 0; $i < $this->_columnCount; $i++) { + $mapper[$i] = $this->_mapperFields[$mapperKeys[$i][0]]; + $mapperKeysMain[$i] = $mapperKeys[$i][0]; + } + + $this->set('mapper', $mapper); + + // store mapping Id to display it in the preview page + $this->set('loadMappingId', CRM_Utils_Array::value('mappingId', $params)); + + //Updating Mapping Records + if (CRM_Utils_Array::value('updateMapping', $params)) { + + $mappingFields = new CRM_Core_DAO_MappingField(); + $mappingFields->mapping_id = $params['mappingId']; + $mappingFields->find(); + + $mappingFieldsId = array(); + while ($mappingFields->fetch()) { + if ($mappingFields->id) { + $mappingFieldsId[$mappingFields->column_number] = $mappingFields->id; + } + } + + for ($i = 0; $i < $this->_columnCount; $i++) { + $updateMappingFields = new CRM_Core_DAO_MappingField(); + $updateMappingFields->id = $mappingFieldsId[$i]; + $updateMappingFields->mapping_id = $params['mappingId']; + $updateMappingFields->column_number = $i; + + $explodedValues = explode('_', $mapperKeys[$i][0]); + $id = CRM_Utils_Array::value(0, $explodedValues); + $first = CRM_Utils_Array::value(1, $explodedValues); + $second = CRM_Utils_Array::value(2, $explodedValues); + + $updateMappingFields->name = $mapper[$i]; + $updateMappingFields->save(); + } + } + + //Saving Mapping Details and Records + if (CRM_Utils_Array::value('saveMapping', $params)) { + $mappingParams = array( + 'name' => $params['saveMappingName'], + 'description' => $params['saveMappingDesc'], + 'mapping_type_id' => CRM_Core_OptionGroup::getValue('mapping_type', + $this->_mappingType, + 'name' + ), + ); + $saveMapping = CRM_Core_BAO_Mapping::add($mappingParams); + + for ($i = 0; $i < $this->_columnCount; $i++) { + $saveMappingFields = new CRM_Core_DAO_MappingField(); + $saveMappingFields->mapping_id = $saveMapping->id; + $saveMappingFields->column_number = $i; + + $explodedValues = explode('_', $mapperKeys[$i][0]); + $id = CRM_Utils_Array::value(0, $explodedValues); + $first = CRM_Utils_Array::value(1, $explodedValues); + $second = CRM_Utils_Array::value(2, $explodedValues); + + $saveMappingFields->name = $mapper[$i]; + $saveMappingFields->save(); + } + $this->set('savedMapping', $saveMappingFields->mapping_id); + } + $this->set('_entity', $this->_entity); + + $parser = new $this->_parser($mapperKeysMain); + $parser->setEntity($this->_multipleCustomData); + $parser->run($fileName, $separator, $mapper, $skipColumnHeader, + CRM_Import_Parser::MODE_PREVIEW, $this->get('contactType') + ); + // add all the necessary variables to the form + $parser->set($this); + } +} \ No newline at end of file diff --git a/CRM/Custom/Import/Form/Preview.php b/CRM/Custom/Import/Form/Preview.php new file mode 100644 index 0000000000..2003975769 --- /dev/null +++ b/CRM/Custom/Import/Form/Preview.php @@ -0,0 +1,145 @@ +controller->exportValue('DataSource', 'skipColumnHeader'); + + //get the data from the session + $dataValues = $this->get('dataValues'); + $mapper = $this->get('mapper'); + $invalidRowCount = $this->get('invalidRowCount'); + $conflictRowCount = $this->get('conflictRowCount'); + $mismatchCount = $this->get('unMatchCount'); + $entity = $this->get('_entity'); + + //get the mapping name displayed if the mappingId is set + $mappingId = $this->get('loadMappingId'); + if ($mappingId) { + $mapDAO = new CRM_Core_DAO_Mapping(); + $mapDAO->id = $mappingId; + $mapDAO->find(TRUE); + $this->assign('loadedMapping', $mappingId); + $this->assign('savedName', $mapDAO->name); + } + + if ($skipColumnHeader) { + $this->assign('skipColumnHeader', $skipColumnHeader); + $this->assign('rowDisplayCount', 3); + } + else { + $this->assign('rowDisplayCount', 2); + } + + if ($invalidRowCount) { + $urlParams = 'type=' . CRM_Import_Parser::ERROR . $this->_importParserUrl; + $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + } + + if ($conflictRowCount) { + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . $this->_importParserUrl; + $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + } + + if ($mismatchCount) { + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . $this->_importParserUrl; + $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + } + + $properties = array( + 'mapper', + 'dataValues', 'columnCount', + 'totalRowCount', 'validRowCount', + 'invalidRowCount', 'conflictRowCount', + 'downloadErrorRecordsUrl', + 'downloadConflictRecordsUrl', + 'downloadMismatchRecordsUrl', + ); + + foreach ($properties as $property) { + $this->assign($property, $this->get($property)); + } + } + + /** + * Process the mapped fields and map it into the uploaded file + * preview the file and extract some summary statistics + * + * @return void + * @access public + */ + public function postProcess() { + $fileName = $this->controller->exportValue('DataSource', 'uploadFile'); + $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); + $invalidRowCount = $this->get('invalidRowCount'); + $conflictRowCount = $this->get('conflictRowCount'); + $onDuplicate = $this->get('onDuplicate'); + $entity = $this->get('_entity'); + + $config = CRM_Core_Config::singleton(); + $separator = $config->fieldSeparator; + + $mapper = $this->controller->exportValue('MapField', 'mapper'); + $mapperKeys = array(); + + foreach ($mapper as $key => $value) { + $mapperKeys[$key] = $mapper[$key][0]; + } + + $parser = new $this->_parser($mapperKeys); + $parser->setEntity($entity); + + $mapFields = $this->get('fields'); + + foreach ($mapper as $key => $value) { + $header = array(); + if (isset($mapFields[$mapper[$key][0]])) { + $header[] = $mapFields[$mapper[$key][0]]; + } + $mapperFields[] = implode(' - ', $header); + } + $parser->run($fileName, $separator, + $mapperFields, + $skipColumnHeader, + CRM_Import_Parser::MODE_IMPORT, + $this->get('contactType'), + $onDuplicate + ); + + // add all the necessary variables to the form + $parser->set($this, CRM_Import_Parser::MODE_IMPORT); + + // check if there is any error occured + + $errorStack = CRM_Core_Error::singleton(); + $errors = $errorStack->getErrors(); + $errorMessage = array(); + + if (is_array($errors)) { + foreach ($errors as $key => $value) { + $errorMessage[] = $value['message']; + } + + $errorFile = $fileName['name'] . '.error.log'; + + if ($fd = fopen($errorFile, 'w')) { + fwrite($fd, implode('\n', $errorMessage)); + } + fclose($fd); + + $this->set('errorFile', $errorFile); + $urlParams = 'type=' . CRM_Import_Parser::ERROR . $this->_importParserUrl; + $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . $this->_importParserUrl; + $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . $this->_importParserUrl; + $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); + } + } +} \ No newline at end of file diff --git a/CRM/Custom/Import/Form/Summary.php b/CRM/Custom/Import/Form/Summary.php new file mode 100644 index 0000000000..90ce14d825 --- /dev/null +++ b/CRM/Custom/Import/Form/Summary.php @@ -0,0 +1,14 @@ +pushUserContext(CRM_Utils_System::url('civicrm/import/custom', 'reset=1')); + } +} \ No newline at end of file diff --git a/CRM/Custom/Import/Parser.php b/CRM/Custom/Import/Parser.php new file mode 100644 index 0000000000..93a40eb759 --- /dev/null +++ b/CRM/Custom/Import/Parser.php @@ -0,0 +1,373 @@ +_contactType = 'Individual'; + break; + + case CRM_Import_Parser::CONTACT_HOUSEHOLD: + $this->_contactType = 'Household'; + break; + + case CRM_Import_Parser::CONTACT_ORGANIZATION: + $this->_contactType = 'Organization'; + } + $this->init(); + + $this->_haveColumnHeader = $skipColumnHeader; + + $this->_separator = $separator; + + $fd = fopen($fileName, "r"); + if (!$fd) { + return FALSE; + } + + $this->_lineCount = $this->_warningCount = 0; + $this->_invalidRowCount = $this->_validCount = 0; + $this->_totalCount = $this->_conflictCount = 0; + + $this->_errors = array(); + $this->_warnings = array(); + $this->_conflicts = array(); + + $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); + + if ($mode == self::MODE_MAPFIELD) { + $this->_rows = array(); + } + else { + $this->_activeFieldCount = count($this->_activeFields); + } + + while (!feof($fd)) { + $this->_lineCount++; + + $values = fgetcsv($fd, 8192, $separator); + if (!$values) { + continue; + } + + self::encloseScrub($values); + + // skip column header if we're not in mapfield mode + if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { + $skipColumnHeader = FALSE; + continue; + } + + /* trim whitespace around the values */ + + $empty = TRUE; + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + + if (CRM_Utils_System::isNull($values)) { + continue; + } + + $this->_totalCount++; + + if ($mode == self::MODE_MAPFIELD) { + $returnCode = $this->mapField($values); + } + elseif ($mode == self::MODE_PREVIEW) { + $returnCode = $this->preview($values); + } + elseif ($mode == self::MODE_SUMMARY) { + $returnCode = $this->summary($values); + } + elseif ($mode == self::MODE_IMPORT) { + $returnCode = $this->import($onDuplicate, $values); + } + else { + $returnCode = self::ERROR; + } + + // note that a line could be valid but still produce a warning + if ($returnCode & self::VALID) { + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode & self::WARNING) { + $this->_warningCount++; + if ($this->_warningCount < $this->_maxWarningCount) { + $this->_warningCount[] = $line; + } + } + + if ($returnCode & self::ERROR) { + $this->_invalidRowCount++; + if ($this->_invalidRowCount < $this->_maxErrorCount) { + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_errors[] = $values; + } + } + + if ($returnCode & self::CONFLICT) { + $this->_conflictCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_conflicts[] = $values; + } + + if ($returnCode & self::DUPLICATE) { + if ($returnCode & self::MULTIPLE_DUPE) { + /* TODO: multi-dupes should be counted apart from singles + * on non-skip action */ + } + $this->_duplicateCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_duplicates[] = $values; + if ($onDuplicate != self::DUPLICATE_SKIP) { + $this->_validCount++; + } + } + + // we give the derived class a way of aborting the process + // note that the return code could be multiple code or'ed together + if ($returnCode & self::STOP) { + break; + } + + // if we are done processing the maxNumber of lines, break + if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { + break; + } + } + + fclose($fd); + + + if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { + $customHeaders = $mapper; + + $customfields = CRM_Core_BAO_CustomField::getFields('Activity'); + foreach ($customHeaders as $key => $value) { + if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { + $customHeaders[$key] = $customfields[$id][0]; + } + } + if ($this->_invalidRowCount) { + // removed view url for invlaid contacts + $headers = array_merge(array(ts('Line Number'), + ts('Reason'), + ), + $customHeaders + ); + $this->_errorFileName = self::errorFileName(self::ERROR); + self::exportCSV($this->_errorFileName, $headers, $this->_errors); + } + if ($this->_conflictCount) { + $headers = array_merge(array(ts('Line Number'), + ts('Reason'), + ), + $customHeaders + ); + $this->_conflictFileName = self::errorFileName(self::CONFLICT); + self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); + } + if ($this->_duplicateCount) { + $headers = array_merge(array(ts('Line Number'), + ts('View Activity History URL'), + ), + $customHeaders + ); + + $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); + self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); + } + } + return $this->fini(); + } + + /** + * Given a list of the importable field keys that the user has selected + * set the active fields array to this list + * + * @param array mapped array of values + * + * @return void + * @access public + */ + function setActiveFields($fieldKeys) { + $this->_activeFieldCount = count($fieldKeys); + foreach ($fieldKeys as $key) { + if (empty($this->_fields[$key])) { + $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -')); + } + else { + $this->_activeFields[] = clone($this->_fields[$key]); + } + } + } + + /** + * function to format the field values for input to the api + * + * @return array (reference ) associative array of name/value pairs + * @access public + */ + function &getActiveFieldParams() { + $params = array(); + for ($i = 0; $i < $this->_activeFieldCount; $i++) { + if (isset($this->_activeFields[$i]->_value) + && !isset($params[$this->_activeFields[$i]->_name]) + && !isset($this->_activeFields[$i]->_related) + ) { + $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; + } + } + return $params; + } + + /** + * Store parser values + * + * @param CRM_Core_Session $store + * + * @return void + * @access public + */ + function set($store, $mode = self::MODE_SUMMARY) { + $store->set('fileSize', $this->_fileSize); + $store->set('lineCount', $this->_lineCount); + $store->set('seperator', $this->_separator); + $store->set('fields', $this->getSelectValues()); + $store->set('fieldTypes', $this->getSelectTypes()); + + $store->set('headerPatterns', $this->getHeaderPatterns()); + $store->set('dataPatterns', $this->getDataPatterns()); + $store->set('columnCount', $this->_activeFieldCount); + $store->set('_entity', $this->_entity); + $store->set('totalRowCount', $this->_totalCount); + $store->set('validRowCount', $this->_validCount); + $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('conflictRowCount', $this->_conflictCount); + + switch ($this->_contactType) { + case 'Individual': + $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); + break; + + case 'Household': + $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); + break; + + case 'Organization': + $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); + } + + if ($this->_invalidRowCount) { + $store->set('errorsFileName', $this->_errorFileName); + } + if ($this->_conflictCount) { + $store->set('conflictsFileName', $this->_conflictFileName); + } + if (isset($this->_rows) && !empty($this->_rows)) { + $store->set('dataValues', $this->_rows); + } + + if ($mode == self::MODE_IMPORT) { + $store->set('duplicateRowCount', $this->_duplicateCount); + if ($this->_duplicateCount) { + $store->set('duplicatesFileName', $this->_duplicateFileName); + } + } + } +} + diff --git a/CRM/Custom/Import/Parser/Api.php b/CRM/Custom/Import/Parser/Api.php new file mode 100644 index 0000000000..205e4a7c01 --- /dev/null +++ b/CRM/Custom/Import/Parser/Api.php @@ -0,0 +1,247 @@ +_mapperKeys = &$mapperKeys; + } + function setFields() { + $customGroupID = $this->_multipleCustomData; + $importableFields = $this->getGroupFieldsForImport($customGroupID, $this); + $this->_fields = array_merge(array('do_not_import' => array('title' => ts('- do not import -')), 'contact_id' => array('title' => ts('Contact ID'))), $importableFields); + } + + /** + * the initializer code, called before the processing + * + * @return void + * @access public + */ + function init() { + $this->setFields(); + $fields = $this->_fields; + $hasLocationType = FALSE; + + foreach ($fields as $name => $field) { + $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT); + $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//'); + $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//'); + $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern'], $hasLocationType); + } + $this->setActiveFields($this->_mapperKeys); + } + + /** + * handle the values in mapField mode + * + * @param array $values the array of values belonging to this line + * + * @return boolean + * @access public + */ + function mapField(&$values) { + return CRM_Import_Parser::VALID; + } + + /** + * handle the values in preview mode + * + * @param array $values the array of values belonging to this line + * + * @return boolean the result of this processing + * @access public + */ + function preview(&$values) { + return $this->summary($values); + } + + /** + * @param array $values the array of values belonging to this line + * + * @return boolean the result of this processing + * It is called from both the preview & the import actions + * (non-PHPdoc) + * @see CRM_Custom_Import_Parser_BaseClass::summary() + */ + function summary(&$values) { + $erroneousField = NULL; + $response = $this->setActiveFieldValues($values, $erroneousField); + $errorRequired = FALSE; + $missingField = ''; + $this->_params = &$this->getActiveFieldParams(); + + $formatted = $this->_params; + $this->_updateWithId = FALSE; + $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE); + + $this->_params = $this->getActiveFieldParams(); + foreach ($this->_requiredFields as $requiredField) { + if (empty($this->_params[$requiredField])) { + $errorRequired = TRUE; + $missingField .= ' ' . $requiredField; + CRM_Contact_Import_Parser_Contact::addToErrorMsg($this->_entity, $requiredField); + } + } + + if ($errorRequired) { + array_unshift($values, ts('Missing required field(s) :') . $missingField); + return CRM_Import_Parser::ERROR; + } + + $errorMessage = NULL; + + $contactType = $this->_contactType ? $this->_contactType : 'Organization'; + CRM_Contact_Import_Parser_Contact::isErrorInCustomData($this->_params , $errorMessage, $contactType, NULL); + + // pseudoconstants + if ($errorMessage) { + $tempMsg = "Invalid value for field(s) : $errorMessage"; + array_unshift($values, $tempMsg); + $errorMessage = NULL; + return CRM_Import_Parser::ERROR; + } + return CRM_Import_Parser::VALID; + } + + /** + * handle the values in import mode + * + * @param int $onDuplicate the code for what action to take on duplicates + * @param array $values the array of values belonging to this line + * + * @return boolean the result of this processing + * @access public + */ + function import($onDuplicate, &$values) { + $response = $this->summary($values); + if ($response != CRM_Import_Parser::VALID) { + $importRecordParams = array( + $statusFieldName => 'INVALID', + "${statusFieldName}Msg" => "Invalid (Error Code: $response)", + ); + return $response; + } + + $this->_updateWithId = FALSE; + $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE); + + $params = $this->getActiveFieldParams(); + $contactType = $this->_contactType ? $this->_contactType : 'Organization'; + $formatted = array( + 'contact_type' => $contactType, + ); + $session = CRM_Core_Session::singleton(); + $dateType = $session->get('dateTypes'); + + $formatted['id'] = $this->_params['contact_id']; + $setDateFields = array_intersect_key($this->_params, array_flip($this->_dateFields)); + + CRM_Contact_Import_Parser_Contact::formatCommonData($this->_params, $formatted, $formatted) ; + foreach($formatted['custom'] as $key => $val) { + $this->_params['custom_'.$key] = $val[-1]['value']; + } + $this->_params['skipRecentView'] = TRUE; + $this->_params['check_permissions'] = TRUE; + $this->_params['entity_id'] = $formatted['id']; + try{ + civicrm_api3('custom_value', 'create', $this->_params); + } + catch(CiviCRM_API3_Exception $e) { + $error = $e->getMessage(); + array_unshift($values, $error); + return CRM_Import_Parser::ERROR; + } + } + + /** + * Format Date params + * + * Although the api will accept any strtotime valid string CiviCRM accepts at least one date format + * not supported by strtotime so we should run this through a conversion + * @param unknown $params + */ + function formatDateParams() { + $session = CRM_Core_Session::singleton(); + $dateType = $session->get('dateTypes'); + $setDateFields = array_intersect_key($this->_params, array_flip($this->_dateFields)); + + foreach ($setDateFields as $key => $value) { + CRM_Utils_Date::convertToDefaultDate($this->_params, $dateType, $key); + $this->_params[$key] = CRM_Utils_Date::processDate($this->_params[$key]); + } + } + + /** + * Set import entity + * @param string $entity + */ + function setEntity($entity) { + $this->_entity = $entity; + $this->_multipleCustomData = $entity; + } + + /** + * the initializer code, called before the processing + * + * @return void + * @access public + */ + function fini() {} + + /** + * Return the field ids and names (with groups) for import purpose. + * + * @param int $id Custom group ID + * + * @return array $importableFields + * + * @access public + * @static + */ + function getGroupFieldsForImport( $id ) { + $importableFields = array(); + $params = array('custom_group_id' => $id); + $allFields = civicrm_api3('custom_field', 'get', $params); + $fields = $allFields['values']; + foreach ($fields as $id => $values) { + $datatype = CRM_Utils_Array::value('data_type', $values); + if ( $datatype == 'File' ) { + continue; + } + /* generate the key for the fields array */ + $key = "custom_$id"; + $regexp = preg_replace('/[.,;:!?]/', '', CRM_Utils_Array::value(0, $values)); + $importableFields[$key] = array( + 'name' => $key, + 'title' => CRM_Utils_Array::value('label', $values), + 'headerPattern' => '/' . preg_quote($regexp, '/') . '/', + 'import' => 1, + 'custom_field_id' => $id, + 'options_per_line' => CRM_Utils_Array::value('options_per_line', $values), + 'data_type' => CRM_Utils_Array::value('data_type', $values), + 'html_type' => CRM_Utils_Array::value('html_type', $values), + 'is_search_range' => CRM_Utils_Array::value('is_search_range', $values), + ); + if (CRM_Utils_Array::value('html_type', $values) == 'Select Date') { + $importableFields[$key]['date_format'] = CRM_Utils_Array::value('date_format', $values); + $importableFields[$key]['time_format'] = CRM_Utils_Array::value('time_format', $values); + $this->_dateFields[] = $key; + } + } + return $importableFields; + } +} \ No newline at end of file diff --git a/CRM/Utils/Type.php b/CRM/Utils/Type.php index 68b0eed2d7..15b6afff18 100644 --- a/CRM/Utils/Type.php +++ b/CRM/Utils/Type.php @@ -164,7 +164,25 @@ class CRM_Utils_Type { // CRM-8925 case 'Country': case 'StateProvince': - if (CRM_Utils_Rule::positiveInteger($data)) { + // Checked for multi valued state/country value + if (is_array($data)) { + $returnData = TRUE; + foreach ($data as $data) { + if (CRM_Utils_Rule::positiveInteger($data) || CRM_Core_DAO::escapeString($data)) { + $returnData = TRUE; + } + else { + $returnData = FALSE; + } + } + if ($returnData) { + return $data; + } + } + elseif (!is_numeric($data) && CRM_Core_DAO::escapeString($data)) { + return $data; + } + elseif (CRM_Utils_Rule::positiveInteger($data)) { return $data; } break; diff --git a/templates/CRM/Custom/Import/Form/DataSource.hlp b/templates/CRM/Custom/Import/Form/DataSource.hlp new file mode 100644 index 0000000000..c35b68df01 --- /dev/null +++ b/templates/CRM/Custom/Import/Form/DataSource.hlp @@ -0,0 +1,33 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.4 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2010 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} +{htxt id="upload-title"} + {ts}Import File Format{/ts} +{/htxt} + +{htxt id="id-onDuplicate-title"} + {ts}Duplicate Matching{/ts} +{/htxt} + diff --git a/templates/CRM/Custom/Import/Form/DataSource.tpl b/templates/CRM/Custom/Import/Form/DataSource.tpl new file mode 100644 index 0000000000..d4152cd8df --- /dev/null +++ b/templates/CRM/Custom/Import/Form/DataSource.tpl @@ -0,0 +1,94 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.4 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2013 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} + +{* API Import Wizard - Step 1 (upload data file) *} +{* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *} + +
+ {* WizardHeader.tpl provides visual display of steps thru the wizard as well as title for current step *} + {include file="CRM/common/WizardHeader.tpl"} + +
+ {ts}The API Import Wizard allows you to easily upload data against any API create method from other applications into CiviCRM.{/ts} + {ts}Files to be imported must be in the 'comma-separated-values' format (CSV) and must contain data needed to match the data to an existing record in your CiviCRM database.{/ts} {help id='upload'} +
+
+
{include file="CRM/common/formButtons.tpl" location="top"}
+ + + + + + + + + + + + + + + + + + + + + + + + + + {include file="CRM/Core/Date.tpl"} + + {if $savedMapping} + + + + + + + {/if} + +
{$form.entity.label}{$form.entity.html}
{$form.uploadFile.label}{$form.uploadFile.html}
+ + {ts}File format must be comma-separated-values (CSV).{/ts} + +
 {ts 1=$uploadSize}Maximum Upload File Size: %1 MB{/ts}
 {$form.skipColumnHeader.html} {$form.skipColumnHeader.label}
+ + {ts}Check this box if the first row of your file consists of field names (Example: "Contact ID", "Participant Role").{/ts} + +
{$form.multipleCustomData.label} + {$form.multipleCustomData.html}
{$form.contactType.label}{$form.contactType.html}
+ + {ts}Select 'Individual' if you are importing custom data for individual persons.{/ts} + {ts}Select 'Organization' or 'Household' if you are importing custom data . (NOTE: Some built-in contact types may not be enabled for your site.){/ts} + +
{if $loadedMapping}{ts}Select a Different Field Mapping{/ts}{else}{ts}Load Saved Field Mapping{/ts}{/if} + {$form.savedMapping.html}
 {ts}Select Saved Mapping, or leave blank to create a new mapping.{/ts}
+
{include file="CRM/common/formButtons.tpl" location="bottom"}
+
+
diff --git a/templates/CRM/Custom/Import/Form/MapField.tpl b/templates/CRM/Custom/Import/Form/MapField.tpl new file mode 100644 index 0000000000..157c9c6603 --- /dev/null +++ b/templates/CRM/Custom/Import/Form/MapField.tpl @@ -0,0 +1,26 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.4 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2013 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} +{include file="CRM/Contact/Import/Form/MapField.tpl"} \ No newline at end of file diff --git a/templates/CRM/Custom/Import/Form/Preview.tpl b/templates/CRM/Custom/Import/Form/Preview.tpl new file mode 100644 index 0000000000..bc6906fb44 --- /dev/null +++ b/templates/CRM/Custom/Import/Form/Preview.tpl @@ -0,0 +1,26 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.4 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2013 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} + {include file="CRM/Contribute/Import/Form/Preview.tpl"} diff --git a/templates/CRM/Custom/Import/Form/Summary.tpl b/templates/CRM/Custom/Import/Form/Summary.tpl new file mode 100644 index 0000000000..05338e694f --- /dev/null +++ b/templates/CRM/Custom/Import/Form/Summary.tpl @@ -0,0 +1,26 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.4 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2013 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} +{include file="CRM/Contact/Import/Form/Summary.tpl"} \ No newline at end of file -- 2.25.1