From 8fd58f564002d72547de6f930f3e70908c4e0973 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Tue, 12 Oct 2021 22:19:45 -0400 Subject: [PATCH] SearchKit - server-side rendering Moves logic for resolving field values from the client to the server-side, which gives more control over formatting, links, and greater consistency of spreadsheet output. --- Civi/Api4/Query/SqlFunction.php | 10 + .../SearchDisplay/AbstractRunAction.php | 285 +++++++++++++++--- .../Api4/Action/SearchDisplay/Download.php | 35 +-- .../crmSearchAdminResultsTable.component.js | 4 +- .../crmSearchAdmin/searchListing/afforms.html | 12 +- .../crmSearchAdmin/searchListing/buttons.html | 4 +- .../crmSearchAdminSearchListing.component.js | 20 +- .../searchListing/displays.html | 12 +- .../crmSearchAdmin/searchListing/tags.html | 2 +- .../ang/crmSearchDisplay/colType/buttons.html | 8 +- .../ang/crmSearchDisplay/colType/field.html | 22 +- .../ang/crmSearchDisplay/colType/include.html | 2 +- .../ang/crmSearchDisplay/colType/links.html | 8 +- .../ang/crmSearchDisplay/colType/menu.html | 16 +- .../traits/searchDisplayBaseTrait.service.js | 64 +--- .../crmSearchDisplayGridItems.html | 8 +- .../crmSearchDisplayListItems.html | 8 +- .../crmSearchDisplayTableBody.html | 2 +- .../traits/searchDisplayTasksTrait.service.js | 10 +- .../v4/SearchDisplay/SearchDownloadTest.php | 2 +- .../api/v4/SearchDisplay/SearchRunTest.php | 36 ++- 21 files changed, 360 insertions(+), 210 deletions(-) diff --git a/Civi/Api4/Query/SqlFunction.php b/Civi/Api4/Query/SqlFunction.php index ea09a42d5d..6380211026 100644 --- a/Civi/Api4/Query/SqlFunction.php +++ b/Civi/Api4/Query/SqlFunction.php @@ -195,6 +195,16 @@ abstract class SqlFunction extends SqlExpression { return static::$category; } + /** + * All functions return 'SqlFunction' as their type. + * + * To get the function name @see SqlFunction::getName() + * @return string + */ + public function getType(): string { + return 'SqlFunction'; + } + /** * @return string */ diff --git a/ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php b/ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php index 75269cefdd..a9e48d58c1 100644 --- a/ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php +++ b/ext/search_kit/Civi/Api4/Action/SearchDisplay/AbstractRunAction.php @@ -65,6 +65,11 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction { */ private $_afform; + /** + * @var array + */ + private $_selectClause; + /** * @param \Civi\Api4\Generic\Result $result * @throws UnauthorizedException @@ -108,61 +113,178 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction { abstract protected function processResult(\Civi\Api4\Generic\Result $result); /** - * Transform each value returned by the API into 'raw' and 'view' properties + * Transforms each row into an array of raw data and an array of formatted columns + * * @param \Civi\Api4\Generic\Result $result - * @return array + * @return array{data: array, columns: array}[] */ protected function formatResult(\Civi\Api4\Generic\Result $result): array { - $select = []; - foreach ($this->savedSearch['api_params']['select'] as $selectExpr) { - $expr = SqlExpression::convert($selectExpr, TRUE); - $item = [ - 'fields' => [], - 'dataType' => $expr->getDataType(), - ]; - foreach ($expr->getFields() as $field) { - $item['fields'][] = $this->getField($field); - } - if (!isset($item['dataType']) && $item['fields']) { - $item['dataType'] = $item['fields'][0]['data_type']; + $rows = []; + foreach ($result as $index => $row) { + $data = $columns = []; + foreach ($this->getSelectClause() as $key => $item) { + $data[$key] = $this->getValue($key, $row, $index); } - $select[$expr->getAlias()] = $item; - } - $formatted = []; - foreach ($result as $index => $data) { - $row = []; - foreach ($select as $key => $item) { - $row[$key] = $this->getValue($key, $data, $item['dataType'], $index); + foreach ($this->display['settings']['columns'] as $column) { + $columns[] = $this->formatColumn($column, $data); } - $formatted[] = $row; + $rows[] = [ + 'data' => $data, + 'columns' => $columns, + ]; } - return $formatted; + return $rows; } /** - * @param $key - * @param $data - * @param $dataType - * @param $index - * @return array + * @param string $key + * @param array $data + * @param int $rowIndex + * @return mixed */ - private function getValue($key, $data, $dataType, $index) { + private function getValue($key, $data, $rowIndex) { // Get value from api result unless this is a pseudo-field which gets a calculated value switch ($key) { case 'result_row_num': - $raw = $index + 1 + ($this->savedSearch['api_params']['offset'] ?? 0); - break; + return $rowIndex + 1 + ($this->savedSearch['api_params']['offset'] ?? 0); case 'user_contact_id': - $raw = \CRM_Core_Session::getLoggedInContactID(); - break; + return \CRM_Core_Session::getLoggedInContactID(); default: - $raw = $data[$key] ?? NULL; + return $data[$key] ?? NULL; + } + } + + /** + * @param $column + * @param $data + * @return array{val: mixed, links: array, edit: array, label: string, title: string, image: array, cssClass: string} + */ + private function formatColumn($column, $data) { + $column += ['rewrite' => NULL, 'label' => NULL]; + $out = $cssClass = []; + switch ($column['type']) { + case 'field': + if (isset($column['image']) && is_array($column['image'])) { + $out['img'] = $this->formatImage($column, $data); + $out['val'] = $this->replaceTokens($column['image']['alt'] ?? NULL, $data, 'view'); + } + elseif ($column['rewrite']) { + $out['val'] = $this->replaceTokens($column['rewrite'], $data, 'view'); + } + else { + $out['val'] = $this->formatViewValue($column['key'], $data[$column['key']] ?? NULL); + } + if ($this->hasValue($column['label']) && (!empty($column['forceLabel']) || $this->hasValue($out['val']))) { + $out['label'] = $this->replaceTokens($column['label'], $data, 'view'); + } + if (isset($column['title']) && strlen($column['title'])) { + $out['title'] = $this->replaceTokens($column['title'], $data, 'view'); + } + if (!empty($column['link']['path'])) { + $out['links'] = $this->formatFieldLinks($column, $data, $out['val']); + } + elseif (!empty($column['editable']) && !$column['rewrite']) { + $out['edit'] = $this->formatEditableColumn($column, $data); + } + break; + + case 'links': + case 'buttons': + case 'menu': + $out = $this->formatLinksColumn($column, $data); + break; + } + if (!empty($column['alignment'])) { + $cssClass[] = $column['alignment']; + } + if ($cssClass) { + $out['cssClass'] = implode(' ', $cssClass); + } + return $out; + } + + /** + * Format a field value as links + * @param $column + * @param $data + * @param $value + * @return array{text: string, url: string, target: string}[] + */ + private function formatFieldLinks($column, $data, $value): array { + $links = []; + if (!empty($column['image'])) { + $value = ['']; + } + foreach ((array) $value as $index => $val) { + $path = $this->replaceTokens($column['link']['path'], $data, 'url', $index); + if ($path) { + $link = [ + 'text' => $val, + 'url' => $this->getUrl($path), + ]; + if (!empty($column['link']['target'])) { + $link['target'] = $column['link']['target']; + } + $links[] = $link; + } } + return $links; + } + + /** + * Format links for a menu/buttons/links column + * @param $column + * @param $data + * @return array{text: string, url: string, target: string, style: string, icon: string}[] + */ + private function formatLinksColumn($column, $data): array { + $out = ['links' => []]; + if (isset($column['text'])) { + $out['text'] = $this->replaceTokens($column['text'], $data, 'view'); + } + foreach ($column['links'] as $item) { + $path = $this->replaceTokens($item['path'], $data, 'url'); + if ($path) { + $link = [ + 'text' => $this->replaceTokens($item['text'] ?? '', $data, 'view'), + 'url' => $this->getUrl($path), + ]; + foreach (['target', 'style', 'icon'] as $prop) { + if (!empty($item[$prop])) { + $link[$prop] = $item[$prop]; + } + } + $out['links'][] = $link; + } + } + return $out; + } + + /** + * @param string $path + * @return string + */ + private function getUrl(string $path) { + if ($path[0] === '/' || strpos($path, 'http://') || strpos($path, 'https://')) { + return $path; + } + // Use absolute urls when downloading spreadsheet + $absolute = $this->getActionName() === 'download'; + return \CRM_Utils_System::url($path, NULL, $absolute, NULL, FALSE); + } + + private function formatEditableColumn($column, $data) { + + } + + private function formatImage($column, $data) { + $tokenExpr = $column['rewrite'] ?: '[' . $column['key'] . ']'; return [ - 'raw' => $raw, - 'view' => $this->formatViewValue($dataType, $raw), + 'url' => $this->replaceTokens($tokenExpr, $data, 'url'), + 'height' => $column['image']['height'] ?? NULL, + 'width' => $column['image']['width'] ?? NULL, ]; } @@ -179,19 +301,82 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction { return $this->_selectQuery->getField($fieldName, FALSE); } + /** + * Returns the select clause enhanced with metadata + * + * @return array + */ + protected function getSelectClause() { + if (!isset($this->_selectClause)) { + $this->_selectClause = []; + foreach ($this->savedSearch['api_params']['select'] as $selectExpr) { + $expr = SqlExpression::convert($selectExpr, TRUE); + $item = [ + 'fields' => [], + 'type' => $expr->getType(), + 'dataType' => $expr->getDataType(), + ]; + foreach ($expr->getFields() as $fieldName) { + $fieldMeta = $this->getField($fieldName); + if ($fieldMeta) { + $item['fields'][] = $fieldMeta; + } + } + if (!isset($item['dataType']) && $item['fields']) { + $item['dataType'] = $item['fields'][0]['data_type']; + } + $this->_selectClause[$expr->getAlias()] = $item; + } + } + return $this->_selectClause; + } + + /** + * @param string $key + * @return array{fields: array, dataType: string}|NULL + */ + protected function getSelectExpression($key) { + return $this->getSelectClause()[$key] ?? NULL; + } + + /** + * @param string $tokenExpr + * @param array $data + * @param string $format view|raw|url + * @param int $index + * @return string + */ + private function replaceTokens($tokenExpr, $data, $format, $index = 0) { + foreach ($this->getTokens($tokenExpr) as $token) { + $val = $data[$token] ?? NULL; + if (isset($val) && $format === 'view') { + $val = $this->formatViewValue($token, $val); + } + $replacement = is_array($val) ? $val[$index] ?? '' : $val; + // A missing token value in a url invalidates it + if ($format === 'url' && (!isset($replacement) || $replacement === '')) { + return NULL; + } + $tokenExpr = str_replace('[' . $token . ']', $replacement, $tokenExpr); + } + return $tokenExpr; + } + /** * Format raw field value according to data type - * @param $dataType + * @param string $key * @param mixed $rawValue * @return array|string */ - protected function formatViewValue($dataType, $rawValue) { + protected function formatViewValue($key, $rawValue) { if (is_array($rawValue)) { - return array_map(function($val) use ($dataType) { - return $this->formatViewValue($dataType, $val); + return array_map(function($val) use ($key) { + return $this->formatViewValue($key, $val); }, $rawValue); } + $dataType = $this->getSelectExpression($key)['dataType'] ?? NULL; + $formatted = $rawValue; switch ($dataType) { @@ -348,9 +533,10 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction { $defaultSort = $this->display['settings']['sort'] ?? []; $currentSort = $this->sort; - // Validate that requested sort fields are part of the SELECT + // Verify requested sort corresponds to sortable columns foreach ($this->sort as $item) { - if (!in_array($item[0], $this->getSelectAliases())) { + $column = array_column($this->display['settings']['columns'], NULL, 'key')[$item[0]] ?? NULL; + if (!$column || (isset($column['sortable']) && !$column['sortable'])) { $currentSort = NULL; } } @@ -396,19 +582,28 @@ abstract class AbstractRunAction extends \Civi\Api4\Generic\AbstractAction { } } // Add fields referenced via token - $tokens = []; - preg_match_all('/\\[([^]]+)\\]/', $possibleTokens, $tokens); + $tokens = $this->getTokens($possibleTokens); // Only add fields not already in SELECT clause - $additions = array_diff(array_merge($additions, $tokens[1]), $existing); + $additions = array_diff(array_merge($additions, $tokens), $existing); // Tokens for aggregated columns start with 'GROUP_CONCAT_' foreach ($additions as $index => $alias) { if (strpos($alias, 'GROUP_CONCAT_') === 0) { $additions[$index] = 'GROUP_CONCAT(' . $this->getJoinFromAlias(explode('_', $alias, 3)[2]) . ') AS ' . $alias; } } + $this->_selectClause = NULL; $apiParams['select'] = array_unique(array_merge($apiParams['select'], $additions)); } + /** + * @param string $str + */ + private function getTokens($str) { + $tokens = []; + preg_match_all('/\\[([^]]+)\\]/', $str, $tokens); + return array_unique($tokens[1]); + } + /** * Given an alias like Contact_Email_01_location_type_id * this will return Contact_Email_01.location_type_id diff --git a/ext/search_kit/Civi/Api4/Action/SearchDisplay/Download.php b/ext/search_kit/Civi/Api4/Action/SearchDisplay/Download.php index 1756e9dc62..432648ff34 100644 --- a/ext/search_kit/Civi/Api4/Action/SearchDisplay/Download.php +++ b/ext/search_kit/Civi/Api4/Action/SearchDisplay/Download.php @@ -71,10 +71,10 @@ class Download extends AbstractRunAction { $rows = $this->formatResult($apiResult); $columns = []; - foreach ($this->display['settings']['columns'] as $col) { + foreach ($this->display['settings']['columns'] as $index => $col) { $col += ['type' => NULL, 'label' => '', 'rewrite' => FALSE]; if ($col['type'] === 'field' && !empty($col['key'])) { - $columns[] = $col; + $columns[$index] = $col; } } @@ -83,12 +83,9 @@ class Download extends AbstractRunAction { switch ($this->format) { case 'array': - $result[] = $columns; + $result[] = array_column($columns, 'label'); foreach ($rows as $data) { - $row = []; - foreach ($columns as $col) { - $row[] = $this->formatColumnValue($col, $data); - } + $row = array_column(array_intersect_key($data['columns'], $columns), 'val'); $result[] = $row; } return; @@ -119,9 +116,11 @@ class Download extends AbstractRunAction { $csv->insertOne(array_column($columns, 'label')); foreach ($rows as $data) { - $row = []; - foreach ($columns as $col) { - $row[] = $this->formatColumnValue($col, $data); + $row = array_column(array_intersect_key($data['columns'], $columns), 'val'); + foreach ($row as &$val) { + if (is_array($val)) { + $val = implode(', ', $val); + } } $csv->insertOne($row); } @@ -141,13 +140,14 @@ class Download extends AbstractRunAction { $sheet = $document->getActiveSheet(); // Header row - foreach ($columns as $index => $col) { + foreach (array_values($columns) as $index => $col) { $sheet->setCellValueByColumnAndRow($index + 1, 1, $col['label']); } foreach ($rows as $rowNum => $data) { + $colNum = 1; foreach ($columns as $index => $col) { - $sheet->setCellValueByColumnAndRow($index + 1, $rowNum + 2, $this->formatColumnValue($col, $data)); + $sheet->setCellValueByColumnAndRow($colNum++, $rowNum + 2, $this->formatColumnValue($col, $data['columns'][$index])); } } @@ -160,16 +160,11 @@ class Download extends AbstractRunAction { * Returns final formatted column value * * @param array $col - * @param array $data + * @param array $value * @return string */ - protected function formatColumnValue(array $col, array $data) { - $val = $col['rewrite'] ?: $data[$col['key']]['view'] ?? ''; - if ($col['rewrite']) { - foreach ($data as $k => $v) { - $val = str_replace("[$k]", $v['view'], $val); - } - } + protected function formatColumnValue(array $col, array $value) { + $val = $value['val'] ?? ''; return is_array($val) ? implode(', ', $val) : $val; } diff --git a/ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js b/ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js index f00dc3a831..b74b81933b 100644 --- a/ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js +++ b/ext/search_kit/ang/crmSearchAdmin/resultsTable/crmSearchAdminResultsTable.component.js @@ -77,6 +77,9 @@ }; ctrl.settings = ctrl.display.settings; setLabel(); + ctrl.results = null; + ctrl.rowCount = null; + ctrl.page = 1; } function setLabel() { @@ -86,7 +89,6 @@ this.$onInit = function() { buildSettings(); this.initializeDisplay($scope, $element); - $scope.$watch('$ctrl.search.api_entity', buildSettings); $scope.$watch('$ctrl.search.api_params', buildSettings, true); $scope.$watch('$ctrl.search.label', setLabel); }; diff --git a/ext/search_kit/ang/crmSearchAdmin/searchListing/afforms.html b/ext/search_kit/ang/crmSearchAdmin/searchListing/afforms.html index 856b6d10d1..211d50a69c 100644 --- a/ext/search_kit/ang/crmSearchAdmin/searchListing/afforms.html +++ b/ext/search_kit/ang/crmSearchAdmin/searchListing/afforms.html @@ -1,22 +1,22 @@ -
+