From 46a7c32a5689419289f68ebec0720d3df54424df Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Tue, 3 May 2022 15:40:32 +1200 Subject: [PATCH] Add extra functions to DataSource This adds functions from https://github.com/civicrm/civicrm-core/pull/23292 to the DataSource class (without them being called as yet) - part of trying to get to the point where fixes are not dependent on each other --- CRM/Import/DataSource.php | 225 +++++++++++++++++++++++++++++++++++--- CRM/Import/Forms.php | 2 +- 2 files changed, 211 insertions(+), 16 deletions(-) diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php index 323f074c04..e9e1c42afe 100644 --- a/CRM/Import/DataSource.php +++ b/CRM/Import/DataSource.php @@ -23,6 +23,68 @@ use Civi\Api4\UserJob; */ abstract class CRM_Import_DataSource { + /** + * @var \CRM_Core_DAO + */ + private $queryResultObject; + + /** + * @var int + */ + private $limit; + + /** + * @param int $limit + * + * @return CRM_Import_DataSource + */ + public function setLimit(int $limit): CRM_Import_DataSource { + $this->limit = $limit; + $this->queryResultObject = NULL; + return $this; + } + + /** + * @param int $offset + * + * @return CRM_Import_DataSource + */ + public function setOffset(int $offset): CRM_Import_DataSource { + $this->offset = $offset; + $this->queryResultObject = NULL; + return $this; + } + + /** + * @var int + */ + private $offset; + + /** + * Statuses of rows to fetch. + * + * @var array + */ + private $statuses = []; + + /** + * Current row. + * + * @var array + */ + private $row; + + /** + * @param array $statuses + * + * @return self + */ + public function setStatuses(array $statuses): self { + $this->statuses = $statuses; + $this->queryResultObject = NULL; + return $this; + } + /** * Class constructor. * @@ -113,33 +175,63 @@ abstract class CRM_Import_DataSource { * * The array has all values. * - * @param int $limit - * @param int $offset + * @param bool $nonAssociative + * Return as a non-associative array? * * @return array * * @throws \API_Exception * @throws \CRM_Core_Exception */ - public function getRows(int $limit = 0, int $offset = 0) { - $query = 'SELECT * FROM ' . $this->getTableName(); - if ($limit) { - $query .= ' LIMIT ' . $limit . ($offset ? (' OFFSET ' . $offset) : NULL); - } + public function getRows(bool $nonAssociative = TRUE): array { $rows = []; - $result = CRM_Core_DAO::executeQuery($query); - while ($result->fetch()) { - $values = $result->toArray(); - /* trim whitespace around the values */ - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } + while ($this->getRow()) { // Historically we expect a non-associative array... - $rows[] = array_values($values); + $rows[] = $nonAssociative ? array_values($this->row) : $this->row; } + $this->queryResultObject = NULL; return $rows; } + /** + * Get the next row. + * + * @return array|null + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + public function getRow(): ?array { + if (!$this->queryResultObject) { + $this->instantiateQueryObject(); + } + if (!$this->queryResultObject->fetch()) { + return NULL; + } + $values = $this->queryResultObject->toArray(); + /* trim whitespace around the values */ + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + $this->row = $values; + return $values; + } + + /** + * Get row count. + * + * The array has all values. + * + * @return int + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + public function getRowCount(array $statuses = []): int { + $this->statuses = $statuses; + $query = 'SELECT count(*) FROM ' . $this->getTableName() . ' ' . $this->getStatusClause(); + return CRM_Core_DAO::singleValueQuery($query); + } + /** * Get an array of column headers, if any. * @@ -284,4 +376,107 @@ abstract class CRM_Import_DataSource { $this->userJob['metadata'] = $metaData; } + /** + * Purge any datasource related assets when the datasource is dropped. + * + * This is the datasource's chance to delete any tables etc that it created + * which will now not be used. + * + * @param array $newParams + * If the dataSource is being updated to another variant of the same + * class (eg. the csv upload was set to no column headers and they + * have resubmitted WITH skipColumnHeader (first row is a header) then + * the dataSource is still CSV and the params for the new intance + * are passed in. When changing from csv to SQL (for example) newParams is + * empty. + * + * @return array + * The details to update the DataSource key in the userJob metadata to. + * Generally and empty array but it the datasource decided (for example) + * that the table it created earlier is still consistent with the new params + * then it might decided not to drop the table and would want to retain + * some metadata. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + * @noinspection PhpUnusedParameterInspection + */ + public function purge(array $newParams = []) :array { + // The old name is still stored... + $oldTableName = $this->getTableName(); + if ($oldTableName) { + CRM_Core_DAO::executeQuery('DROP TABLE IF EXISTS ' . $oldTableName); + } + return []; + } + + /** + * Add a status columns to the import table. + * + * We add + * _id - primary key + * _status + * _statusMsg + * + * Note that + * 1) the use of the preceding underscore has 2 purposes - it avoids clashing + * with an id field (code comments from 14 years ago suggest perhaps there + * could be cases where it still clashes but time didn't tell in this case) + * 2) the show fields query used to get the column names excluded the + * administrative fields, relying on this convention. + * 3) we have the capitalisation on _statusMsg - @todo change to _status_message + * + * @param string $tableName + */ + protected function addTrackingFieldsToTable(string $tableName): void { + CRM_Core_DAO::executeQuery(" + ALTER TABLE $tableName + ADD COLUMN _status VARCHAR(32) DEFAULT 'NEW' NOT NULL, + ADD COLUMN _statusMsg TEXT, + ADD COLUMN _id INT PRIMARY KEY NOT NULL AUTO_INCREMENT" + ); + } + + /** + * + * @throws \API_Exception + * @throws \CRM_Core_Exception + */ + private function instantiateQueryObject(): void { + $query = 'SELECT * FROM ' . $this->getTableName() . ' ' . $this->getStatusClause(); + if ($this->limit) { + $query .= ' LIMIT ' . $this->limit . ($this->offset ? (' OFFSET ' . $this->offset) : NULL); + } + $this->queryResultObject = CRM_Core_DAO::executeQuery($query); + } + + /** + * Get the mapping of constants to database status codes. + * + * @return string[] + */ + protected function getStatusMapping() { + return [ + CRM_Import_Parser::VALID => 'imported', + CRM_Import_Parser::ERROR => 'error', + CRM_Import_Parser::DUPLICATE => 'duplicate', + ]; + } + + /** + * Get the status filter clause. + * + * @return string + */ + private function getStatusClause(): string { + if (!empty($this->statuses)) { + $statuses = []; + foreach ($this->statuses as $status) { + $statuses[] = '"' . $this->getStatusMapping()[$status] . '"'; + } + return ' WHERE _status IN (' . implode(',', $statuses) . ')'; + } + return ''; + } + } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 9375b636dd..eadc650477 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -416,7 +416,7 @@ class CRM_Import_Forms extends CRM_Core_Form { * @throws \API_Exception */ protected function getDataRows(int $limit): array { - return $this->getDataSourceObject()->getRows($limit); + return $this->getDataSourceObject()->setLimit($limit)->getRows(); } /** -- 2.25.1