$dao = new CRM_Core_DAO();
$db = $dao->getDatabaseConnection();
$dataSource->postProcess($this->_params, $db, $this);
- $this->updateUserJobMetadata('DataSource', $dataSource->getDataSourceMetadata());
}
/**
/**
* Set variables up before form is built.
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ * @throws \Civi\API\Exception\UnauthorizedException
*/
public function preProcess() {
- $dataSource = $this->get('dataSource');
- $skipColumnHeader = $this->get('skipColumnHeader');
$this->_mapperFields = $this->get('fields');
$this->_importTableName = $this->get('importTableName');
$this->_onDuplicate = $this->get('onDuplicate');
$this->assign('highlightedFields', $highlightedFields);
$this->_formattedFieldNames[$contactType] = $this->_mapperFields = array_merge($this->_mapperFields, $formattedFieldNames);
- $columnNames = [];
- //get original col headers from csv if present.
- if ($dataSource == 'CRM_Import_DataSource_CSV' && $skipColumnHeader) {
- $columnNames = $this->get('originalColHeader');
- }
- else {
- // get the field names from the temp. DB table
- $columnsQuery = "SHOW FIELDS FROM $this->_importTableName
- WHERE Field NOT LIKE '\_%'";
- $columnsResult = CRM_Core_DAO::executeQuery($columnsQuery);
- while ($columnsResult->fetch()) {
- $columnNames[] = $columnsResult->Field;
- }
- }
-
- $showColNames = TRUE;
- if ($dataSource === 'CRM_Import_DataSource_CSV' && !$skipColumnHeader) {
- $showColNames = FALSE;
- }
- $this->assign('showColNames', $showColNames);
+ $columnNames = $this->getColumnHeaders();
+ $this->assign('showColNames', !empty($columnNames));
- $this->_columnCount = count($columnNames);
+ $this->_columnCount = $this->getNumberOfColumns();
$this->_columnNames = $columnNames;
- $this->assign('columnNames', $columnNames);
+ $this->assign('columnNames', $this->getColumnHeaders());
//$this->_columnCount = $this->get( 'columnCount' );
$this->assign('columnCount', $this->_columnCount);
- $this->_dataValues = $this->get('dataValues');
+ $this->_dataValues = array_values($this->getDataRows(2));
$this->assign('dataValues', $this->_dataValues);
- $this->assign('rowDisplayCount', 2);
}
/**
foreach ($mapperKeys as $key) {
// check if there is a _a_b or _b_a in the key
if (strpos($key, '_a_b') || strpos($key, '_b_a')) {
- list($id, $first, $second) = explode('_', $key);
+ [$id, $first, $second] = explode('_', $key);
}
else {
$id = $first = $second = NULL;
}
//relationship contact mapper info.
- list($id, $first, $second) = CRM_Utils_System::explode('_', $fldName, 3);
+ [$id, $first, $second] = CRM_Utils_System::explode('_', $fldName, 3);
if (($first === 'a' && $second === 'b') ||
($first === 'b' && $second === 'a')
) {
}
$mappingID = NULL;
- for ($i = 0; $i < $this->_columnCount; $i++) {
+ foreach (array_keys($this->getColumnHeaders()) as $i) {
$mappingID = $this->saveMappingField($mapperKeys, $saveMapping, $cType, $i, $mapper, $parserParameters);
}
$this->set('savedMapping', $mappingID);
$parserParameters['relatedContactPhoneType'], $parserParameters['relatedContactImProvider'],
$parserParameters['mapperWebsiteType'], $parserParameters['relatedContactWebsiteType']
);
+ $parser->setUserJobID($this->getUserJobID());
- $primaryKeyName = $this->get('primaryKeyName');
- $statusFieldName = $this->get('statusFieldName');
$parser->run($this->_importTableName,
$mapper,
CRM_Import_Parser::MODE_PREVIEW,
$this->get('contactType'),
- $primaryKeyName,
- $statusFieldName,
+ '_id',
+ '_status',
$this->_onDuplicate,
NULL, NULL, FALSE,
CRM_Contact_Import_Parser_Contact::DEFAULT_TIMEOUT,
* Set variables up before form is built.
*/
public function preProcess() {
- //get the data from the session
- $dataValues = $this->get('dataValues');
$mapper = $this->get('mapper');
$invalidRowCount = $this->get('invalidRowCount');
$conflictRowCount = $this->get('conflictRowCount');
'locations',
'phones',
'ims',
- 'dataValues',
'columnCount',
'totalRowCount',
'validRowCount',
foreach ($properties as $property) {
$this->assign($property, $this->get($property));
}
+ $this->assign('dataValues', $this->getDataRows(2));
$this->setStatusUrl();
$this->_mapperRelatedContactWebsiteType = $mapperRelatedContactWebsiteType;
// get IM service provider type id for related contact
$this->_mapperRelatedContactImProvider = &$mapperRelatedContactImProvider;
+ $this->setFieldMetadata();
+ foreach ($this->getImportableFieldsMetadata() as $name => $field) {
+ $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field));
+ }
}
/**
* The initializer code, called before processing.
*/
public function init() {
- $this->setFieldMetadata();
- foreach ($this->getImportableFieldsMetadata() as $name => $field) {
- $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field));
- }
-
$this->_newContacts = [];
$this->setActiveFields($this->_mapperKeys);
$this->_conflicts = [];
$this->_unparsedAddresses = [];
+ // Transitional support for deprecating table_name (and other fields)
+ // form input - the goal is to load them from userJob - but eventually
+ // we will just load the datasource object and this code will not know the
+ // table name.
+ if (!$tableName && $this->userJobID) {
+ $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
+ }
+
$this->_tableName = $tableName;
- $this->_primaryKeyName = $primaryKeyName;
- $this->_statusFieldName = $statusFieldName;
+ $this->_primaryKeyName = '_id';
+ $this->_statusFieldName = '_status';
if ($mode == self::MODE_MAPFIELD) {
$this->_rows = [];
return $this->userJob;
}
+ /**
+ * Get submitted value.
+ *
+ * Get a value submitted on the form.
+ *
+ * @return mixed
+ *
+ * @throws \API_Exception
+ */
+ protected function getSubmittedValue(string $valueName) {
+ return $this->getUserJob()['metadata']['submitted_values'][$valueName];
+ }
+
+ /**
+ * Get rows as an array.
+ *
+ * The array has all values.
+ *
+ * @param int $limit
+ * @param int $offset
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ public function getRows(int $limit = 0, int $offset = 0) {
+ $query = 'SELECT * FROM ' . $this->getTableName();
+ if ($limit) {
+ $query .= ' LIMIT ' . $limit . ($offset ? (' OFFSET ' . $offset) : NULL);
+ }
+ $rows = [];
+ $result = CRM_Core_DAO::executeQuery($query);
+ while ($result->fetch()) {
+ $values = $result->toArray();
+ /* trim whitespace around the values */
+ foreach ($values as $k => $v) {
+ $values[$k] = trim($v, " \t\r\n");
+ }
+ // Historically we expect a non-associative array...
+ $rows[] = array_values($values);
+ }
+ return $rows;
+ }
+
+ /**
+ * Get an array of column headers, if any.
+ *
+ * Null is returned when there are none - ie because a csv file does not
+ * have an initial header row.
+ *
+ * This is presented to the user in the MapField screen so
+ * that can see what fields they are mapping.
+ *
+ * @return array
+ * @throws \API_Exception
+ */
+ public function getColumnHeaders(): array {
+ return $this->getUserJob()['metadata']['DataSource']['column_headers'];
+ }
+
+ /**
+ * Get an array of column headers, if any.
+ *
+ * Null is returned when there are none - ie because a csv file does not
+ * have an initial header row.
+ *
+ * This is presented to the user in the MapField screen so
+ * that can see what fields they are mapping.
+ *
+ * @return int
+ * @throws \API_Exception
+ */
+ public function getNumberOfColumns(): int {
+ return $this->getUserJob()['metadata']['DataSource']['number_of_columns'];
+ }
+
/**
* Generated metadata relating to the the datasource.
*
protected $dataSourceMetadata = [];
/**
+ * Get metadata about the datasource.
+ *
* @return array
+ *
+ * @throws \API_Exception
*/
public function getDataSourceMetadata(): array {
+ if (!$this->dataSourceMetadata && $this->getUserJobID()) {
+ $this->dataSourceMetadata = $this->getUserJob()['metadata']['DataSource'];
+ }
+
return $this->dataSourceMetadata;
}
+ /**
+ * Get the table name for the datajob.
+ *
+ * @return string|null
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ protected function getTableName(): ?string {
+ // The old name is still stored...
+ $tableName = $this->getDataSourceMetadata()['table_name'];
+ if (!$tableName) {
+ return NULL;
+ }
+ if (strpos($tableName, 'civicrm_tmp_') !== 0
+ || !CRM_Utils_Rule::alphanumeric($tableName)) {
+ // The table name is generated and stored by code, not users so it
+ // should be safe - but a check seems prudent all the same.
+ throw new CRM_Core_Exception('Table cannot be deleted');
+ }
+ return $tableName;
+ }
+
/**
* Get the fields declared for this datasource.
*
return empty($info['permissions']) || CRM_Core_Permission::check($info['permissions']);
}
+ /**
+ * @param string $key
+ * @param array $data
+ *
+ * @throws \API_Exception
+ * @throws \Civi\API\Exception\UnauthorizedException
+ */
+ protected function updateUserJobMetadata(string $key, array $data): void {
+ $metaData = array_merge(
+ $this->getUserJob()['metadata'],
+ [$key => $data]
+ );
+ UserJob::update(FALSE)
+ ->addWhere('id', '=', $this->getUserJobID())
+ ->setValues(['metadata' => $metaData])
+ ->execute();
+ $this->userJob['metadata'] = $metaData;
+ }
+
}
* @param string $db
* @param \CRM_Core_Form $form
*
+ * @throws \API_Exception
* @throws \CRM_Core_Exception
*/
public function postProcess(&$params, &$db, &$form) {
$file = $params['uploadFile']['name'];
+ $firstRowIsColumnHeader = $params['skipColumnHeader'] ?? FALSE;
$result = self::_CsvToTable(
$file,
- CRM_Utils_Array::value('skipColumnHeader', $params, FALSE),
+ $firstRowIsColumnHeader,
CRM_Utils_Array::value('import_table_name', $params),
CRM_Utils_Array::value('fieldSeparator', $params, ',')
);
$form->set('originalColHeader', CRM_Utils_Array::value('column_headers', $result));
$form->set('importTableName', $result['import_table_name']);
- $this->dataSourceMetadata = [
+ $this->updateUserJobMetadata('DataSource', [
'table_name' => $result['import_table_name'],
- 'column_headers' => $result['column_headers'] ?? NULL,
- ];
+ 'column_headers' => $firstRowIsColumnHeader ? $result['column_headers'] : [],
+ 'number_of_columns' => $result['number_of_columns'],
+ ]);
}
/**
//get the import tmp table name.
$result['import_table_name'] = $tableName;
+ $result['number_of_columns'] = $numColumns;
return $result;
}
$errors = [];
// Makeshift query validation (case-insensitive regex matching on word boundaries)
- $forbidden = ['ALTER', 'CREATE', 'DELETE', 'DESCRIBE', 'DROP', 'SHOW', 'UPDATE', 'information_schema'];
+ $forbidden = ['ALTER', 'CREATE', 'DELETE', 'DESCRIBE', 'DROP', 'SHOW', 'UPDATE', 'REPLACE', 'information_schema'];
foreach ($forbidden as $pattern) {
if (preg_match("/\\b$pattern\\b/i", $fields['sqlQuery'])) {
$errors['sqlQuery'] = ts('The query contains the forbidden %1 command.', [1 => $pattern]);
}
}
- return $errors ? $errors : TRUE;
+ return $errors ?: TRUE;
}
/**
* @param string $db
* @param \CRM_Core_Form $form
*
+ * @throws \API_Exception
* @throws \CRM_Core_Exception
+ * @throws \Civi\API\Exception\UnauthorizedException
*/
public function postProcess(&$params, &$db, &$form) {
$importJob = new CRM_Contact_Import_ImportJob(
);
$form->set('importTableName', $importJob->getTableName());
- $this->dataSourceMetadata = [
+ // Get the names of the fields to be imported. Any fields starting with an
+ // underscore are considered to be internal to the import process)
+ $columnsResult = CRM_Core_DAO::executeQuery(
+ 'SHOW FIELDS FROM ' . $importJob->getTableName() . "
+ WHERE Field NOT LIKE '\_%'");
+
+ $columnNames = [];
+ while ($columnsResult->fetch()) {
+ $columnNames[] = $columnsResult->Field;
+ }
+ $this->updateUserJobMetadata('DataSource', [
'table_name' => $importJob->getTableName(),
- ];
+ 'column_headers' => $columnNames,
+ 'number_of_columns' => count($columnNames),
+ ]);
}
}
$this->userJob['metadata'] = $metaData;
}
+ /**
+ * Get column headers for the datasource or empty array if none apply.
+ *
+ * This would be the first row of a csv or the fields in an sql query.
+ *
+ * If the csv does not have a header row it will be empty.
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ protected function getColumnHeaders(): array {
+ return $this->getDataSourceObject()->getColumnHeaders();
+ }
+
+ /**
+ * Get the number of importable columns in the data source.
+ *
+ * @return int
+ *
+ * @throws \API_Exception
+ * @throws \CRM_Core_Exception
+ */
+ protected function getNumberOfColumns(): int {
+ return $this->getDataSourceObject()->getNumberOfColumns();
+ }
+
+ /**
+ * Get x data rows from the datasource.
+ *
+ * At this stage we are fetching from what has been stored in the form
+ * during `postProcess` on the DataSource form.
+ *
+ * In the future we will use the dataSource object, likely
+ * supporting offset as well.
+ *
+ * @param int $limit
+ *
+ * @return array
+ *
+ * @throws \CRM_Core_Exception
+ * @throws \API_Exception
+ */
+ protected function getDataRows(int $limit): array {
+ return $this->getDataSourceObject()->getRows($limit);
+ }
+
}
</div>
<div class="crm-submit-buttons">{include file="CRM/common/formButtons.tpl" location="top"}</div>
{* Table for mapping data to CRM fields *}
- {include file="CRM/Contact/Import/Form/MapTable.tpl}
+ {include file="CRM/Contact/Import/Form/MapTable.tpl" mapper=$form.mapper}
<script type="text/javascript" >
{literal}
{if $savedMappingName}
<tr class="columnheader-dark"><th colspan="4">{ts 1=$savedMappingName}Saved Field Mapping: %1{/ts}</td></tr>
{/if}
- <tr class="columnheader">
- {if $showColNames}
- {assign var="totalRowsDisplay" value=$rowDisplayCount+1}
- {else}
- {assign var="totalRowsDisplay" value=$rowDisplayCount}
- {/if}
- {section name=rows loop=$totalRowsDisplay}
- {if $smarty.section.rows.iteration == 1 and $showColNames}
- <td>{ts}Column Names{/ts}</td>
- {elseif $showColNames}
- <td>{ts 1=$smarty.section.rows.iteration-1}Import Data (row %1){/ts}</td>
- {else}
- <td>{ts 1=$smarty.section.rows.iteration}Import Data (row %1){/ts}</td>
- {/if}
- {/section}
- <td>{ts}Matching CiviCRM Field{/ts}</td>
- </tr>
+ {* Header row - has column for column names if they have been supplied *}
+ <tr class="columnheader">
+ {if $columnNames}
+ <td>{ts}Column Names{/ts}</td>
+ {/if}
+ {foreach from=$dataValues item=row key=index}
+ {math equation="x + y" x=$index y=1 assign="rowNumber"}
+ <td>{ts 1=$rowNumber}Import Data (row %1){/ts}</td>
+ {/foreach}
+ <td>{ts}Matching CiviCRM Field{/ts}</td>
+ </tr>
{*Loop on columns parsed from the import data rows*}
- {section name=cols loop=$columnCount}
- {assign var="i" value=$smarty.section.cols.index}
+ {foreach from=$mapper key=i item=mapperField}
+
<tr style="border: 1px solid #DDDDDD;">
- {if $showColNames}
- <td class="even-row labels">{$columnNames[$i]}</td>
+ {if array_key_exists($i, $columnNames)}
+ <td class="even-row labels">{$columnNames[$i]}</td>
{/if}
-
- {section name=rows loop=$rowDisplayCount}
- {assign var="j" value=$smarty.section.rows.index}
- <td class="odd-row">{$dataValues[$j][$i]|escape}</td>
- {/section}
+ {foreach from=$dataValues item=row key=index}
+ <td class="odd-row">{$row[$i]|escape}</td>
+ {/foreach}
{* Display mapper <select> field for 'Map Fields', and mapper value for 'Preview' *}
<td class="form-item even-row{if $wizard.currentStepName == 'Preview'} labels{/if}">
- {if $wizard.currentStepName == 'Preview'}
+ {if $wizard.currentStepName == 'Preview'}
{if $relatedContactDetails && $relatedContactDetails[$i] != ''}
{$mapper[$i]} - {$relatedContactDetails[$i]}
{$mapper[$i]}
{*/if*}
{/if}
- {else}
- {$form.mapper[$i].html|smarty:nodefaults}
- {/if}
+ {else}
+ {$mapperField.html|smarty:nodefaults}
+ {/if}
</td>
</tr>
- {/section}
+ {/foreach}
</table>
{/strip}
* File for the CRM_Contact_Import_Form_MapFieldTest class.
*/
+use Civi\Api4\UserJob;
+
/**
* Test contact import map field.
*
* @param array $mapper
* @param array $expecteds
*
+ * @throws \API_Exception
* @throws \CRM_Core_Exception
* @throws \CiviCRM_API3_Exception
*/
- public function testSubmit($params, $mapper, $expecteds = []) {
- CRM_Core_DAO::executeQuery('CREATE TABLE IF NOT EXISTS civicrm_import_job_xxx (`nada` text, `first_name` text, `last_name` text, `address` text) ENGINE=InnoDB DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci');
- $form = $this->getFormObject('CRM_Contact_Import_Form_MapField');
+ public function testSubmit($params, $mapper, $expecteds = []): void {
+ $form = $this->getMapFieldFormObject('CRM_Contact_Import_Form_MapField');
/* @var CRM_Contact_Import_Form_MapField $form */
$form->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
$form->_columnNames = ['nada', 'first_name', 'last_name', 'address'];
- $form->set('importTableName', 'civicrm_import_job_xxx');
$form->preProcess();
$form->submit($params, $mapper);
- CRM_Core_DAO::executeQuery("DROP TABLE civicrm_import_job_xxx");
+ CRM_Core_DAO::executeQuery('DROP TABLE civicrm_tmp_d_import_job_xxx');
if (!empty($expecteds)) {
foreach ($expecteds as $expected) {
$result = $this->callAPISuccess($expected['entity'], 'get', array_merge($expected['values'], ['sequential' => 1]));
}
/**
- * Instantiate form object
+ * Instantiate MapField form object
*
- * @param string $class
- * @param array $formValues
- * @param string $pageName
- * @param array $searchFormValues
- * Values for the search form if the form is a task eg.
- * for selected ids 6 & 8:
- * [
- * 'radio_ts' => 'ts_sel',
- * 'task' => CRM_Member_Task::PDF_LETTER,
- * 'mark_x_6' => 1,
- * 'mark_x_8' => 1,
- * ]
- *
- * @return \CRM_Core_Form
+ * @return \CRM_Contact_Import_Form_MapField
+ * @throws \API_Exception
* @throws \CRM_Core_Exception
*/
- public function getFormObject($class, $formValues = [], $pageName = '', $searchFormValues = []) {
- $form = parent::getFormObject($class);
+ public function getMapFieldFormObject(): CRM_Contact_Import_Form_MapField {
+ CRM_Core_DAO::executeQuery('CREATE TABLE IF NOT EXISTS civicrm_tmp_d_import_job_xxx (`nada` text, `first_name` text, `last_name` text, `address` text) ENGINE=InnoDB DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci');
+ $userJobID = UserJob::create()->setValues([
+ 'metadata' => [
+ 'submitted_values' => [
+ 'dataSource' => 'CRM_Import_DataSource_SQL',
+ 'sqlQuery' => 'SELECT * FROM civicrm_tmp_d_import_job_xxx',
+ ],
+ ],
+ 'status_id:name' => 'draft',
+ 'type_id:name' => 'contact_import',
+ ])->execute()->first()['id'];
+
+ $dataSource = new CRM_Import_DataSource_SQL($userJobID);
+ $params = ['sqlQuery' => 'SELECT * FROM civicrm_tmp_d_import_job_xxx'];
+ $null = NULL;
+ $form = $this->getFormObject('CRM_Contact_Import_Form_MapField');
+ $form->set('user_job_id', $userJobID);
+ $dataSource->postProcess($params, $null, $form);
+
$contactFields = CRM_Contact_BAO_Contact::importableFields();
$fields = [];
foreach ($contactFields as $name => $field) {
* @throws \API_Exception
* @throws \CRM_Core_Exception
* @throws \CiviCRM_API3_Exception
- * @throws \Civi\API\Exception\UnauthorizedException
*/
- public function testLoadSavedMappingDirect() {
+ public function testLoadSavedMappingDirect(): void {
$this->entity = 'Contact';
$this->createCustomGroupWithFieldOfType(['title' => 'My Field']);
$this->setUpMapFieldForm();
* @throws \CRM_Core_Exception
*/
private function setUpMapFieldForm() {
- $this->form = $this->getFormObject('CRM_Contact_Import_Form_MapField');
+ $this->form = $this->getMapFieldFormObject();
$this->form->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
}
projects / civicrm-core.git / commitdiff