From 39e0f675c2c57586670cb495f785e5e65720d626 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Wed, 15 Jan 2020 20:59:59 -0500 Subject: [PATCH] Api4 - Support wildcard * in select clause --- Civi/Api4/Generic/AbstractAction.php | 3 +- Civi/Api4/Generic/AbstractGetAction.php | 21 ++++- Civi/Api4/Generic/BasicGetAction.php | 1 + Civi/Api4/Generic/DAOGetAction.php | 12 +++ Civi/Api4/Query/Api4SelectQuery.php | 48 ++++++---- .../Schema/Joinable/CustomGroupJoinable.php | 3 + .../Api4/Service/Schema/Joinable/Joinable.php | 11 +++ Civi/Api4/Utils/SelectUtil.php | 60 +++++++++++++ ang/api4Explorer/Explorer.html | 2 +- ang/api4Explorer/Explorer.js | 14 ++- api/api.php | 2 +- .../api/v4/Action/GetFromArrayTest.php | 2 +- .../phpunit/api/v4/Entity/ContactJoinTest.php | 2 +- .../api/v4/Mock/Api4/MockArrayEntity.php | 9 +- .../Query/Api4SelectQueryComplexJoinTest.php | 7 +- tests/phpunit/api/v4/Utils/SelectUtilTest.php | 90 +++++++++++++++++++ 16 files changed, 257 insertions(+), 30 deletions(-) create mode 100644 Civi/Api4/Utils/SelectUtil.php create mode 100644 tests/phpunit/api/v4/Utils/SelectUtilTest.php diff --git a/Civi/Api4/Generic/AbstractAction.php b/Civi/Api4/Generic/AbstractAction.php index d81c9aa33a..965c5fc724 100644 --- a/Civi/Api4/Generic/AbstractAction.php +++ b/Civi/Api4/Generic/AbstractAction.php @@ -383,12 +383,13 @@ abstract class AbstractAction implements \ArrayAccess { /** * Returns schema fields for this entity & action. * - * Here we bypass the api wrapper and execute the getFields action directly. + * Here we bypass the api wrapper and run the getFields action directly. * This is because we DON'T want the wrapper to check permissions as this is an internal op, * but we DO want permissions to be checked inside the getFields request so e.g. the api_key * field can be conditionally included. * @see \Civi\Api4\Action\Contact\GetFields * + * @throws \API_Exception * @return array */ public function entityFields() { diff --git a/Civi/Api4/Generic/AbstractGetAction.php b/Civi/Api4/Generic/AbstractGetAction.php index e48b3c1597..adf84fb43b 100644 --- a/Civi/Api4/Generic/AbstractGetAction.php +++ b/Civi/Api4/Generic/AbstractGetAction.php @@ -21,6 +21,8 @@ namespace Civi\Api4\Generic; +use Civi\Api4\Utils\SelectUtil; + /** * Base class for all "Get" api actions. * @@ -33,7 +35,10 @@ namespace Civi\Api4\Generic; abstract class AbstractGetAction extends AbstractQueryAction { /** - * Fields to return. Defaults to all fields. + * Fields to return. Defaults to all fields ["*"]. + * + * Use the * wildcard by itself to select all available fields, or use it to match similarly-named fields. + * E.g. "is_*" will match fields named is_primary, is_active, etc. * * Set to ["row_count"] to return only the number of items found. * @@ -70,6 +75,20 @@ abstract class AbstractGetAction extends AbstractQueryAction { } } + /** + * Adds all fields matched by the * wildcard + * + * @throws \API_Exception + */ + protected function expandSelectClauseWildcards() { + foreach ($this->select as $item) { + if (strpos($item, '*') !== FALSE && strpos($item, '.') === FALSE) { + $this->select = array_diff($this->select, [$item]); + $this->select = array_unique(array_merge($this->select, SelectUtil::getMatchingFields($item, array_column($this->entityFields(), 'name')))); + } + } + } + /** * Helper to parse the WHERE param for getRecords to perform simple pre-filtering. * diff --git a/Civi/Api4/Generic/BasicGetAction.php b/Civi/Api4/Generic/BasicGetAction.php index 2de0146eb0..a9357bb21d 100644 --- a/Civi/Api4/Generic/BasicGetAction.php +++ b/Civi/Api4/Generic/BasicGetAction.php @@ -57,6 +57,7 @@ class BasicGetAction extends AbstractGetAction { */ public function _run(Result $result) { $this->setDefaultWhereClause(); + $this->expandSelectClauseWildcards(); $values = $this->getRecords(); $result->exchangeArray($this->queryArray($values)); } diff --git a/Civi/Api4/Generic/DAOGetAction.php b/Civi/Api4/Generic/DAOGetAction.php index 728506b7d0..a85682e480 100644 --- a/Civi/Api4/Generic/DAOGetAction.php +++ b/Civi/Api4/Generic/DAOGetAction.php @@ -31,8 +31,20 @@ namespace Civi\Api4\Generic; class DAOGetAction extends AbstractGetAction { use Traits\DAOActionTrait; + /** + * Fields to return. Defaults to all non-custom fields ["*"]. + * + * Use the dot notation to perform joins in the select clause, e.g. selecting ["*", "contact.*"] from Email.get + * will select all fields for the email + all fields for the related contact. + * + * @var array + * @inheritDoc + */ + protected $select = []; + public function _run(Result $result) { $this->setDefaultWhereClause(); + $this->expandSelectClauseWildcards(); $result->exchangeArray($this->getObjects()); } diff --git a/Civi/Api4/Query/Api4SelectQuery.php b/Civi/Api4/Query/Api4SelectQuery.php index 80ecaf55cf..664325fc23 100644 --- a/Civi/Api4/Query/Api4SelectQuery.php +++ b/Civi/Api4/Query/Api4SelectQuery.php @@ -18,6 +18,7 @@ use Civi\Api4\Service\Schema\Joinable\CustomGroupJoinable; use Civi\Api4\Service\Schema\Joinable\Joinable; use Civi\Api4\Utils\FormattingUtil; use Civi\Api4\Utils\CoreUtil; +use Civi\Api4\Utils\SelectUtil; use CRM_Core_DAO_AllCoreTables as AllCoreTables; use CRM_Utils_Array as UtilsArray; @@ -161,15 +162,8 @@ class Api4SelectQuery extends SelectQuery { * @throws \Civi\API\Exception\UnauthorizedException */ protected function buildSelectFields() { - $return_all_fields = (empty($this->select) || !is_array($this->select)); - $return = $return_all_fields ? $this->entityFieldNames : $this->select; - if ($return_all_fields || in_array('custom', $this->select)) { - foreach (array_keys($this->apiFieldSpec) as $fieldName) { - if (strpos($fieldName, 'custom_') === 0) { - $return[] = $fieldName; - } - } - } + $selectAll = (empty($this->select) || in_array('*', $this->select)); + $select = $selectAll ? $this->entityFieldNames : $this->select; // Always select the ID if the table has one. if (array_key_exists('id', $this->apiFieldSpec) || strstr($this->entity, 'Custom_')) { @@ -177,7 +171,7 @@ class Api4SelectQuery extends SelectQuery { } // core return fields - foreach ($return as $fieldName) { + foreach ($select as $fieldName) { $field = $this->getField($fieldName); if (strpos($fieldName, '.') && !empty($this->fkSelectAliases[$fieldName]) && !array_filter($this->getPathJoinTypes($fieldName))) { $this->selectFields[$this->fkSelectAliases[$fieldName]] = $fieldName; @@ -338,6 +332,14 @@ class Api4SelectQuery extends SelectQuery { /** @var \Civi\Api4\Service\Schema\Joinable\Joinable $lastLink */ $lastLink = array_pop($joinPath); + $isWild = strpos($field, '*') !== FALSE; + if ($isWild) { + if (!in_array($key, $this->select)) { + throw new \API_Exception('Wildcards can only be used in the SELECT clause.'); + } + $this->select = array_diff($this->select, [$key]); + } + // Cache field info for retrieval by $this->getField() $prefix = array_pop($pathArray) . '.'; if (!isset($this->apiFieldSpec[$prefix . $field])) { @@ -351,19 +353,29 @@ class Api4SelectQuery extends SelectQuery { } } - if (!$lastLink->getField($field)) { + if (!$isWild && !$lastLink->getField($field)) { throw new \API_Exception('Invalid join'); } - // custom groups use aliases for field names - if ($lastLink instanceof CustomGroupJoinable) { - $field = $lastLink->getSqlColumn($field); + $fields = $isWild ? [] : [$field]; + // Expand wildcard and add matching fields to $this->select + if ($isWild) { + $fields = SelectUtil::getMatchingFields($field, $lastLink->getEntityFieldNames()); + foreach ($fields as $field) { + $this->select[] = $pathString . '.' . $field; + } + $this->select = array_unique($this->select); } - // Check Permission on field. - if ($this->checkPermissions && !empty($this->apiFieldSpec[$prefix . $field]['permission']) && !\CRM_Core_Permission::check($this->apiFieldSpec[$prefix . $field]['permission'])) { - return; + + foreach ($fields as $field) { + // custom groups use aliases for field names + $col = ($lastLink instanceof CustomGroupJoinable) ? $lastLink->getSqlColumn($field) : $field; + // Check Permission on field. + if ($this->checkPermissions && !empty($this->apiFieldSpec[$prefix . $field]['permission']) && !\CRM_Core_Permission::check($this->apiFieldSpec[$prefix . $field]['permission'])) { + return; + } + $this->fkSelectAliases[$pathString . '.' . $field] = sprintf('%s.%s', $lastLink->getAlias(), $col); } - $this->fkSelectAliases[$key] = sprintf('%s.%s', $lastLink->getAlias(), $field); } /** diff --git a/Civi/Api4/Service/Schema/Joinable/CustomGroupJoinable.php b/Civi/Api4/Service/Schema/Joinable/CustomGroupJoinable.php index 3e57abfbb0..695c7b5662 100644 --- a/Civi/Api4/Service/Schema/Joinable/CustomGroupJoinable.php +++ b/Civi/Api4/Service/Schema/Joinable/CustomGroupJoinable.php @@ -86,6 +86,9 @@ class CustomGroupJoinable extends Joinable { * @return string */ public function getSqlColumn($fieldName) { + if (strpos($fieldName, '.') !== FALSE) { + $fieldName = substr($fieldName, 1 + strrpos($fieldName, '.')); + } return $this->columns[$fieldName]; } diff --git a/Civi/Api4/Service/Schema/Joinable/Joinable.php b/Civi/Api4/Service/Schema/Joinable/Joinable.php index a0b6338bef..3cd51f970d 100644 --- a/Civi/Api4/Service/Schema/Joinable/Joinable.php +++ b/Civi/Api4/Service/Schema/Joinable/Joinable.php @@ -281,6 +281,17 @@ class Joinable { return $this->entityFields; } + /** + * @return array + */ + public function getEntityFieldNames() { + $fieldNames = []; + foreach ($this->getEntityFields() as $fieldSpec) { + $fieldNames[] = $fieldSpec->getName(); + } + return $fieldNames; + } + /** * @return \Civi\Api4\Service\Spec\FieldSpec|NULL */ diff --git a/Civi/Api4/Utils/SelectUtil.php b/Civi/Api4/Utils/SelectUtil.php new file mode 100644 index 0000000000..38b085ceef --- /dev/null +++ b/Civi/Api4/Utils/SelectUtil.php @@ -0,0 +1,60 @@ +
- +
diff --git a/ang/api4Explorer/Explorer.js b/ang/api4Explorer/Explorer.js index 2bfc82c9e4..b8953ef982 100644 --- a/ang/api4Explorer/Explorer.js +++ b/ang/api4Explorer/Explorer.js @@ -26,6 +26,7 @@ $scope.actions = actions; $scope.fields = []; $scope.fieldsAndJoins = []; + $scope.selectFieldsAndJoins = []; $scope.availableParams = {}; $scope.params = {}; $scope.index = ''; @@ -113,16 +114,17 @@ return fields; } - function addJoins(fieldList) { + function addJoins(fieldList, addWildcard) { var fields = _.cloneDeep(fieldList), fks = _.findWhere(links, {entity: $scope.entity}) || {}; _.each(fks.links, function(link) { - var linkFields = entityFields(link.entity); + var linkFields = _.cloneDeep(entityFields(link.entity)), + wildCard = addWildcard ? [{id: link.alias + '.*', text: link.alias + '.*', 'description': 'All core ' + link.entity + ' fields'}] : []; if (linkFields) { fields.push({ text: link.alias, description: 'Join to ' + link.entity, - children: formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.') + children: wildCard.concat(formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.')) }); } }); @@ -261,7 +263,8 @@ function selectAction() { $scope.action = $routeParams.api4action; - $scope.fieldsAndJoins = []; + $scope.fieldsAndJoins.length = 0; + $scope.selectFieldsAndJoins.length = 0; if (!actions.length) { formatForSelect2(getEntity().actions, actions, 'name', ['description', 'params']); } @@ -270,9 +273,12 @@ $scope.fields = getFieldList($scope.action); if (_.contains(['get', 'update', 'delete', 'replace'], $scope.action)) { $scope.fieldsAndJoins = addJoins($scope.fields); + $scope.selectFieldsAndJoins = addJoins($scope.fields, true); } else { $scope.fieldsAndJoins = $scope.fields; + $scope.selectFieldsAndJoins = _.cloneDeep($scope.fields); } + $scope.selectFieldsAndJoins.unshift({id: '*', text: '*', 'description': 'All core ' + $scope.entity + ' fields'}); _.each(actionInfo.params, function (param, name) { var format, defaultVal = _.cloneDeep(param.default); diff --git a/api/api.php b/api/api.php index cccad8b017..c5cae1e27f 100644 --- a/api/api.php +++ b/api/api.php @@ -63,7 +63,7 @@ function civicrm_api4(string $entity, string $action, array $params = [], $index $removeIndexField = FALSE; // If index field is not part of the select query, we add it here and remove it below - if ($indexField && !empty($params['select']) && is_array($params['select']) && !in_array($indexField, $params['select'])) { + if ($indexField && !empty($params['select']) && is_array($params['select']) && !\Civi\Api4\Utils\SelectUtil::isFieldSelected($indexField, $params['select'])) { $params['select'][] = $indexField; $removeIndexField = TRUE; } diff --git a/tests/phpunit/api/v4/Action/GetFromArrayTest.php b/tests/phpunit/api/v4/Action/GetFromArrayTest.php index 324dfa11d9..74eae3c752 100644 --- a/tests/phpunit/api/v4/Action/GetFromArrayTest.php +++ b/tests/phpunit/api/v4/Action/GetFromArrayTest.php @@ -61,7 +61,7 @@ class GetFromArrayTest extends UnitTestCase { public function testArrayGetWithSelect() { $result = MockArrayEntity::get() ->addSelect('field1') - ->addSelect('field3') + ->addSelect('f*3') ->setLimit(4) ->execute(); $this->assertEquals([ diff --git a/tests/phpunit/api/v4/Entity/ContactJoinTest.php b/tests/phpunit/api/v4/Entity/ContactJoinTest.php index 8492f2caa8..b0ac14c98e 100644 --- a/tests/phpunit/api/v4/Entity/ContactJoinTest.php +++ b/tests/phpunit/api/v4/Entity/ContactJoinTest.php @@ -56,7 +56,7 @@ class ContactJoinTest extends UnitTestCase { foreach ($entitiesToTest as $entity) { $results = civicrm_api4($entity, 'get', [ 'where' => [['contact_id', '=', $contact['id']]], - 'select' => ['contact.display_name', 'contact.id'], + 'select' => ['contact.*_name', 'contact.id'], ]); foreach ($results as $result) { $this->assertEquals($contact['id'], $result['contact.id']); diff --git a/tests/phpunit/api/v4/Mock/Api4/MockArrayEntity.php b/tests/phpunit/api/v4/Mock/Api4/MockArrayEntity.php index 85b413bf1d..62e6bf28f7 100644 --- a/tests/phpunit/api/v4/Mock/Api4/MockArrayEntity.php +++ b/tests/phpunit/api/v4/Mock/Api4/MockArrayEntity.php @@ -34,7 +34,14 @@ class MockArrayEntity extends Generic\AbstractEntity { public static function getFields() { return new BasicGetFieldsAction(static::class, __FUNCTION__, function() { - return []; + return [ + ['name' => 'field1'], + ['name' => 'field2'], + ['name' => 'field3'], + ['name' => 'field4'], + ['name' => 'field5'], + ['name' => 'field6'], + ]; }); } diff --git a/tests/phpunit/api/v4/Query/Api4SelectQueryComplexJoinTest.php b/tests/phpunit/api/v4/Query/Api4SelectQueryComplexJoinTest.php index 846c04c51d..3b08d4577c 100644 --- a/tests/phpunit/api/v4/Query/Api4SelectQueryComplexJoinTest.php +++ b/tests/phpunit/api/v4/Query/Api4SelectQueryComplexJoinTest.php @@ -49,7 +49,7 @@ class Api4SelectQueryComplexJoinTest extends UnitTestCase { $query = new Api4SelectQuery('Contact', FALSE, civicrm_api4('Contact', 'getFields', ['includeCustom' => FALSE, 'checkPermissions' => FALSE, 'action' => 'get'], 'name')); $query->select[] = 'id'; $query->select[] = 'display_name'; - $query->select[] = 'phones.phone'; + $query->select[] = 'phones.*_id'; $query->select[] = 'emails.email'; $query->select[] = 'emails.location_type.name'; $query->select[] = 'created_activities.contact_id'; @@ -75,6 +75,11 @@ class Api4SelectQueryComplexJoinTest extends UnitTestCase { $this->assertArrayHasKey('activity_type', $firstActivity); $activityType = $firstActivity['activity_type']; $this->assertArrayHasKey('name', $activityType); + + $this->assertArrayHasKey('name', $firstResult['emails'][0]['location_type']); + $this->assertArrayHasKey('location_type_id', $firstResult['phones'][0]); + $this->assertArrayHasKey('id', $firstResult['phones'][0]); + $this->assertArrayNotHasKey('phone', $firstResult['phones'][0]); } public function testWithSelectOfOrphanDeepValues() { diff --git a/tests/phpunit/api/v4/Utils/SelectUtilTest.php b/tests/phpunit/api/v4/Utils/SelectUtilTest.php new file mode 100644 index 0000000000..662d343cbc --- /dev/null +++ b/tests/phpunit/api/v4/Utils/SelectUtilTest.php @@ -0,0 +1,90 @@ +assertEquals($expected, SelectUtil::isFieldSelected($field, $selects)); + } + + public function getMatchingExamples() { + return [ + [$this->emailFieldNames, '*'], + [[], 'nothing'], + [['email'], 'email'], + [['contact_id', 'location_type_id'], '*_id'], + [['contact_id', 'location_type_id'], '*o*_id'], + [['contact_id'], 'con*_id'], + [['is_primary', 'is_billing', 'is_bulkmail'], 'is_*'], + [['is_billing', 'is_bulkmail'], 'is_*l*'], + ]; + } + + /** + * @dataProvider getMatchingExamples + * @param $expected + * @param $pattern + */ + public function testGetMatchingFields($expected, $pattern) { + $this->assertEquals($expected, SelectUtil::getMatchingFields($pattern, $this->emailFieldNames)); + } + +} -- 2.25.1