Merge pull request #17719 from civicrm/5.27
[civicrm-core.git] / Civi / Api4 / Query / Api4SelectQuery.php
1 <?php
2 /*
3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
5 | |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
10 */
11
12 namespace Civi\Api4\Query;
13
14 use Civi\Api4\Service\Schema\Joinable\CustomGroupJoinable;
15 use Civi\Api4\Utils\FormattingUtil;
16 use Civi\Api4\Utils\CoreUtil;
17 use Civi\Api4\Utils\SelectUtil;
18
19 /**
20 * A query `node` may be in one of three formats:
21 *
22 * * leaf: [$fieldName, $operator, $criteria]
23 * * negated: ['NOT', $node]
24 * * branch: ['OR|NOT', [$node, $node, ...]]
25 *
26 * Leaf operators are one of:
27 *
28 * * '=', '<=', '>=', '>', '<', 'LIKE', "<>", "!=",
29 * * "NOT LIKE", 'IN', 'NOT IN', 'BETWEEN', 'NOT BETWEEN',
30 * * 'IS NOT NULL', or 'IS NULL'.
31 */
32 class Api4SelectQuery {
33
34 const
35 MAIN_TABLE_ALIAS = 'a',
36 UNLIMITED = '18446744073709551615';
37
38 /**
39 * @var \CRM_Utils_SQL_Select
40 */
41 protected $query;
42
43 /**
44 * @var array
45 */
46 protected $joins = [];
47
48 /**
49 * @var array[]
50 */
51 protected $apiFieldSpec;
52
53 /**
54 * @var array
55 */
56 protected $entityFieldNames = [];
57
58 /**
59 * @var array
60 */
61 protected $aclFields = [];
62
63 /**
64 * @var \Civi\Api4\Generic\DAOGetAction
65 */
66 private $api;
67
68 /**
69 * @var array
70 * [alias => expr][]
71 */
72 protected $selectAliases = [];
73
74 /**
75 * @var bool
76 */
77 public $forceSelectId = TRUE;
78
79 /**
80 * @param \Civi\Api4\Generic\DAOGetAction $apiGet
81 */
82 public function __construct($apiGet) {
83 $this->api = $apiGet;
84
85 // Always select ID of main table unless grouping by something else
86 $this->forceSelectId = !$this->getGroupBy() || $this->getGroupBy() === ['id'];
87
88 // Build field lists
89 foreach ($this->api->entityFields() as $field) {
90 $this->entityFieldNames[] = $field['name'];
91 $field['sql_name'] = '`' . self::MAIN_TABLE_ALIAS . '`.`' . $field['column_name'] . '`';
92 $this->addSpecField($field['name'], $field);
93 }
94
95 $tableName = CoreUtil::getTableName($this->getEntity());
96 $this->query = \CRM_Utils_SQL_Select::from($tableName . ' ' . self::MAIN_TABLE_ALIAS);
97
98 // Add ACLs first to avoid redundant subclauses
99 $baoName = CoreUtil::getBAOFromApiName($this->getEntity());
100 $this->query->where($this->getAclClause(self::MAIN_TABLE_ALIAS, $baoName));
101 }
102
103 /**
104 * Builds main final sql statement after initialization.
105 *
106 * @return string
107 * @throws \API_Exception
108 * @throws \CRM_Core_Exception
109 */
110 public function getSql() {
111 // Add explicit joins. Other joins implied by dot notation may be added later
112 $this->addExplicitJoins();
113 $this->buildSelectClause();
114 $this->buildWhereClause();
115 $this->buildOrderBy();
116 $this->buildLimit();
117 $this->buildGroupBy();
118 $this->buildHavingClause();
119 return $this->query->toSQL();
120 }
121
122 /**
123 * Why walk when you can
124 *
125 * @return array
126 */
127 public function run() {
128 $results = [];
129 $sql = $this->getSql();
130 $this->debug('sql', $sql);
131 $query = \CRM_Core_DAO::executeQuery($sql);
132 while ($query->fetch()) {
133 $result = [];
134 foreach ($this->selectAliases as $alias => $expr) {
135 $returnName = $alias;
136 $alias = str_replace('.', '_', $alias);
137 $result[$returnName] = property_exists($query, $alias) ? $query->$alias : NULL;
138 }
139 $results[] = $result;
140 }
141 FormattingUtil::formatOutputValues($results, $this->apiFieldSpec, $this->getEntity());
142 return $results;
143 }
144
145 /**
146 * @return int
147 * @throws \API_Exception
148 */
149 public function getCount() {
150 $this->addExplicitJoins();
151 $this->buildWhereClause();
152 // If no having or groupBy, we only need to select count
153 if (!$this->getHaving() && !$this->getGroupBy()) {
154 $this->query->select('COUNT(*) AS `c`');
155 $sql = $this->query->toSQL();
156 }
157 // Use a subquery to count groups from GROUP BY or results filtered by HAVING
158 else {
159 // With no HAVING, just select the last field grouped by
160 if (!$this->getHaving()) {
161 $select = array_slice($this->getGroupBy(), -1);
162 }
163 $this->buildSelectClause($select ?? NULL);
164 $this->buildHavingClause();
165 $this->buildGroupBy();
166 $subquery = $this->query->toSQL();
167 $sql = "SELECT count(*) AS `c` FROM ( $subquery ) AS rows";
168 }
169 $this->debug('sql', $sql);
170 return (int) \CRM_Core_DAO::singleValueQuery($sql);
171 }
172
173 /**
174 * @param array $select
175 * Array of select expressions; defaults to $this->getSelect
176 * @throws \API_Exception
177 */
178 protected function buildSelectClause($select = NULL) {
179 // Use default if select not provided, exclude row_count which is handled elsewhere
180 $select = array_diff($select ?? $this->getSelect(), ['row_count']);
181 // An empty select is the same as *
182 if (empty($select)) {
183 $select = $this->entityFieldNames;
184 }
185 else {
186 if ($this->forceSelectId) {
187 $select = array_merge(['id'], $select);
188 }
189
190 // Expand wildcards in joins (the api wrapper already expanded non-joined wildcards)
191 $wildFields = array_filter($select, function($item) {
192 return strpos($item, '*') !== FALSE && strpos($item, '.') !== FALSE && strpos($item, '(') === FALSE && strpos($item, ' ') === FALSE;
193 });
194 foreach ($wildFields as $item) {
195 $pos = array_search($item, array_values($select));
196 $this->autoJoinFK($item);
197 $matches = SelectUtil::getMatchingFields($item, array_keys($this->apiFieldSpec));
198 array_splice($select, $pos, 1, $matches);
199 }
200 $select = array_unique($select);
201 }
202 foreach ($select as $item) {
203 $expr = SqlExpression::convert($item, TRUE);
204 $valid = TRUE;
205 foreach ($expr->getFields() as $fieldName) {
206 $field = $this->getField($fieldName);
207 // Remove expressions with unknown fields without raising an error
208 if (!$field) {
209 $select = array_diff($select, [$item]);
210 $this->debug('undefined_fields', $fieldName);
211 $valid = FALSE;
212 }
213 }
214 if ($valid) {
215 $alias = $expr->getAlias();
216 if ($alias != $expr->getExpr() && isset($this->apiFieldSpec[$alias])) {
217 throw new \API_Exception('Cannot use existing field name as alias');
218 }
219 $this->selectAliases[$alias] = $expr->getExpr();
220 $this->query->select($expr->render($this->apiFieldSpec) . " AS `$alias`");
221 }
222 }
223 }
224
225 /**
226 * Add WHERE clause to query
227 */
228 protected function buildWhereClause() {
229 foreach ($this->getWhere() as $clause) {
230 $sql = $this->treeWalkClauses($clause, 'WHERE');
231 if ($sql) {
232 $this->query->where($sql);
233 }
234 }
235 }
236
237 /**
238 * Add HAVING clause to query
239 *
240 * Every expression referenced must also be in the SELECT clause.
241 */
242 protected function buildHavingClause() {
243 foreach ($this->getHaving() as $clause) {
244 $this->query->having($this->treeWalkClauses($clause, 'HAVING'));
245 }
246 }
247
248 /**
249 * Add ORDER BY to query
250 */
251 protected function buildOrderBy() {
252 foreach ($this->getOrderBy() as $item => $dir) {
253 if ($dir !== 'ASC' && $dir !== 'DESC') {
254 throw new \API_Exception("Invalid sort direction. Cannot order by $item $dir");
255 }
256 $expr = $this->getExpression($item);
257 $column = $expr->render($this->apiFieldSpec);
258
259 // Use FIELD() function to sort on pseudoconstant values
260 $suffix = strstr($item, ':');
261 if ($suffix && $expr->getType() === 'SqlField') {
262 $field = $this->getField($item);
263 $options = FormattingUtil::getPseudoconstantList($field['entity'], $field['name'], substr($suffix, 1));
264 if ($options) {
265 asort($options);
266 $column = "FIELD($column,'" . implode("','", array_keys($options)) . "')";
267 }
268 }
269 $this->query->orderBy("$column $dir");
270 }
271 }
272
273 /**
274 * Add LIMIT to query
275 *
276 * @throws \CRM_Core_Exception
277 */
278 protected function buildLimit() {
279 if ($this->getLimit() || $this->getOffset()) {
280 // If limit is 0, mysql will actually return 0 results. Instead set to maximum possible.
281 $this->query->limit($this->getLimit() ?: self::UNLIMITED, $this->getOffset());
282 }
283 }
284
285 /**
286 * Add GROUP BY clause to query
287 */
288 protected function buildGroupBy() {
289 foreach ($this->getGroupBy() as $item) {
290 $this->query->groupBy($this->getExpression($item)->render($this->apiFieldSpec));
291 }
292 }
293
294 /**
295 * Recursively validate and transform a branch or leaf clause array to SQL.
296 *
297 * @param array $clause
298 * @param string $type
299 * WHERE|HAVING|ON
300 * @return string SQL where clause
301 *
302 * @throws \API_Exception
303 * @uses composeClause() to generate the SQL etc.
304 */
305 protected function treeWalkClauses($clause, $type) {
306 // Skip empty leaf.
307 if (in_array($clause[0], ['AND', 'OR', 'NOT']) && empty($clause[1])) {
308 return '';
309 }
310 switch ($clause[0]) {
311 case 'OR':
312 case 'AND':
313 // handle branches
314 if (count($clause[1]) === 1) {
315 // a single set so AND|OR is immaterial
316 return $this->treeWalkClauses($clause[1][0], $type);
317 }
318 else {
319 $sql_subclauses = [];
320 foreach ($clause[1] as $subclause) {
321 $sql_subclauses[] = $this->treeWalkClauses($subclause, $type);
322 }
323 return '(' . implode("\n" . $clause[0], $sql_subclauses) . ')';
324 }
325
326 case 'NOT':
327 // If we get a group of clauses with no operator, assume AND
328 if (!is_string($clause[1][0])) {
329 $clause[1] = ['AND', $clause[1]];
330 }
331 return 'NOT (' . $this->treeWalkClauses($clause[1], $type) . ')';
332
333 default:
334 return $this->composeClause($clause, $type);
335 }
336 }
337
338 /**
339 * Validate and transform a leaf clause array to SQL.
340 * @param array $clause [$fieldName, $operator, $criteria]
341 * @param string $type
342 * WHERE|HAVING|ON
343 * @return string SQL
344 * @throws \API_Exception
345 * @throws \Exception
346 */
347 protected function composeClause(array $clause, string $type) {
348 // Pad array for unary operators
349 list($expr, $operator, $value) = array_pad($clause, 3, NULL);
350 if (!in_array($operator, \CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
351 throw new \API_Exception('Illegal operator');
352 }
353
354 // For WHERE clause, expr must be the name of a field.
355 if ($type === 'WHERE') {
356 $field = $this->getField($expr, TRUE);
357 FormattingUtil::formatInputValue($value, $expr, $field);
358 $fieldAlias = $field['sql_name'];
359 }
360 // For HAVING, expr must be an item in the SELECT clause
361 elseif ($type === 'HAVING') {
362 // Expr references a fieldName or alias
363 if (isset($this->selectAliases[$expr])) {
364 $fieldAlias = $expr;
365 // Attempt to format if this is a real field
366 if (isset($this->apiFieldSpec[$expr])) {
367 FormattingUtil::formatInputValue($value, $expr, $this->apiFieldSpec[$expr]);
368 }
369 }
370 // Expr references a non-field expression like a function; convert to alias
371 elseif (in_array($expr, $this->selectAliases)) {
372 $fieldAlias = array_search($expr, $this->selectAliases);
373 }
374 // If either the having or select field contains a pseudoconstant suffix, match and perform substitution
375 else {
376 list($fieldName) = explode(':', $expr);
377 foreach ($this->selectAliases as $selectAlias => $selectExpr) {
378 list($selectField) = explode(':', $selectAlias);
379 if ($selectAlias === $selectExpr && $fieldName === $selectField && isset($this->apiFieldSpec[$fieldName])) {
380 FormattingUtil::formatInputValue($value, $expr, $this->apiFieldSpec[$fieldName]);
381 $fieldAlias = $selectAlias;
382 break;
383 }
384 }
385 }
386 if (!isset($fieldAlias)) {
387 throw new \API_Exception("Invalid expression in HAVING clause: '$expr'. Must use a value from SELECT clause.");
388 }
389 $fieldAlias = '`' . $fieldAlias . '`';
390 }
391 elseif ($type === 'ON') {
392 $expr = $this->getExpression($expr);
393 $fieldName = count($expr->getFields()) === 1 ? $expr->getFields()[0] : NULL;
394 $fieldAlias = $expr->render($this->apiFieldSpec);
395 if (is_string($value)) {
396 $valExpr = $this->getExpression($value);
397 if ($fieldName && $valExpr->getType() === 'SqlString') {
398 FormattingUtil::formatInputValue($valExpr->expr, $fieldName, $this->apiFieldSpec[$fieldName]);
399 }
400 return sprintf('%s %s %s', $fieldAlias, $operator, $valExpr->render($this->apiFieldSpec));
401 }
402 elseif ($fieldName) {
403 FormattingUtil::formatInputValue($value, $fieldName, $this->apiFieldSpec[$fieldName]);
404 }
405 }
406
407 $sql_clause = \CRM_Core_DAO::createSQLFilter($fieldAlias, [$operator => $value]);
408 if ($sql_clause === NULL) {
409 throw new \API_Exception("Invalid value in $type clause for '$expr'");
410 }
411 return $sql_clause;
412 }
413
414 /**
415 * @param string $expr
416 * @return SqlExpression
417 * @throws \API_Exception
418 */
419 protected function getExpression(string $expr) {
420 $sqlExpr = SqlExpression::convert($expr);
421 foreach ($sqlExpr->getFields() as $fieldName) {
422 $this->getField($fieldName, TRUE);
423 }
424 return $sqlExpr;
425 }
426
427 /**
428 * Get acl clause for an entity
429 *
430 * @param string $tableAlias
431 * @param \CRM_Core_DAO|string $baoName
432 * @param array $stack
433 * @return array
434 */
435 public function getAclClause($tableAlias, $baoName, $stack = []) {
436 if (!$this->getCheckPermissions()) {
437 return [];
438 }
439 // Prevent (most) redundant acl sub clauses if they have already been applied to the main entity.
440 // FIXME: Currently this only works 1 level deep, but tracking through multiple joins would increase complexity
441 // and just doing it for the first join takes care of most acl clause deduping.
442 if (count($stack) === 1 && in_array($stack[0], $this->aclFields)) {
443 return [];
444 }
445 $clauses = $baoName::getSelectWhereClause($tableAlias);
446 if (!$stack) {
447 // Track field clauses added to the main entity
448 $this->aclFields = array_keys($clauses);
449 }
450 return array_filter($clauses);
451 }
452
453 /**
454 * Fetch a field from the getFields list
455 *
456 * @param string $expr
457 * @param bool $strict
458 * In strict mode, this will throw an exception if the field doesn't exist
459 *
460 * @return array|null
461 * @throws \API_Exception
462 */
463 public function getField($expr, $strict = FALSE) {
464 // If the expression contains a pseudoconstant filter like activity_type_id:label,
465 // strip it to look up the base field name, then add the field:filter key to apiFieldSpec
466 $col = strpos($expr, ':');
467 $fieldName = $col ? substr($expr, 0, $col) : $expr;
468 // Perform join if field not yet available - this will add it to apiFieldSpec
469 if (!isset($this->apiFieldSpec[$fieldName]) && strpos($fieldName, '.')) {
470 $this->autoJoinFK($fieldName);
471 }
472 $field = $this->apiFieldSpec[$fieldName] ?? NULL;
473 if ($strict && !$field) {
474 throw new \API_Exception("Invalid field '$fieldName'");
475 }
476 $this->apiFieldSpec[$expr] = $field;
477 return $field;
478 }
479
480 /**
481 * Join onto other entities as specified by the api call.
482 *
483 * @throws \API_Exception
484 * @throws \Civi\API\Exception\NotImplementedException
485 */
486 private function addExplicitJoins() {
487 foreach ($this->getJoin() as $join) {
488 // First item in the array is the entity name
489 $entity = array_shift($join);
490 // Which might contain an alias. Split on the keyword "AS"
491 list($entity, $alias) = array_pad(explode(' AS ', $entity), 2, NULL);
492 // Ensure alias is a safe string, and supply default if not given
493 $alias = $alias ? \CRM_Utils_String::munge($alias) : strtolower($entity);
494 // First item in the array is a boolean indicating if the join is required (aka INNER or LEFT).
495 // The rest are join conditions.
496 $side = array_shift($join) ? 'INNER' : 'LEFT';
497 $joinEntityGet = \Civi\API\Request::create($entity, 'get', ['version' => 4, 'checkPermissions' => $this->getCheckPermissions()]);
498 foreach ($joinEntityGet->entityFields() as $field) {
499 $field['sql_name'] = '`' . $alias . '`.`' . $field['column_name'] . '`';
500 $this->addSpecField($alias . '.' . $field['name'], $field);
501 }
502 $conditions = $this->getJoinConditions($entity, $alias);
503 foreach (array_filter($join) as $clause) {
504 $conditions[] = $this->treeWalkClauses($clause, 'ON');
505 }
506 $tableName = CoreUtil::getTableName($entity);
507 $this->join($side, $tableName, $alias, $conditions);
508 }
509 }
510
511 /**
512 * Supply conditions for an explicit join.
513 *
514 * @param $entity
515 * @param $alias
516 * @return array
517 */
518 private function getJoinConditions($entity, $alias) {
519 $conditions = [];
520 // getAclClause() expects a stack of 1-to-1 join fields to help it dedupe, but this is more flexible,
521 // so unless this is a direct 1-to-1 join with the main entity, we'll just hack it
522 // with a padded empty stack to bypass its deduping.
523 $stack = [NULL, NULL];
524 foreach ($this->apiFieldSpec as $name => $field) {
525 if ($field['entity'] !== $entity && $field['fk_entity'] === $entity) {
526 $conditions[] = $this->treeWalkClauses([$name, '=', "$alias.id"], 'ON');
527 }
528 elseif (strpos($name, "$alias.") === 0 && substr_count($name, '.') === 1 && $field['fk_entity'] === $this->getEntity()) {
529 $conditions[] = $this->treeWalkClauses([$name, '=', 'id'], 'ON');
530 $stack = ['id'];
531 }
532 }
533 // Hmm, if we came up with > 1 condition, then it's ambiguous how it should be joined so we won't return anything but the generic ACLs
534 if (count($conditions) > 1) {
535 $stack = [NULL, NULL];
536 $conditions = [];
537 }
538 $baoName = CoreUtil::getBAOFromApiName($entity);
539 $acls = array_values($this->getAclClause($alias, $baoName, $stack));
540 return array_merge($acls, $conditions);
541 }
542
543 /**
544 * Joins a path and adds all fields in the joined entity to apiFieldSpec
545 *
546 * @param $key
547 * @throws \API_Exception
548 * @throws \Exception
549 */
550 protected function autoJoinFK($key) {
551 if (isset($this->apiFieldSpec[$key])) {
552 return;
553 }
554
555 $pathArray = explode('.', $key);
556
557 /** @var \Civi\Api4\Service\Schema\Joiner $joiner */
558 $joiner = \Civi::container()->get('joiner');
559 // The last item in the path is the field name. We don't care about that; we'll add all fields from the joined entity.
560 array_pop($pathArray);
561 $pathString = implode('.', $pathArray);
562
563 if (!$joiner->canAutoJoin($this->getFrom(), $pathString)) {
564 return;
565 }
566
567 $joinPath = $joiner->join($this, $pathString);
568
569 $lastLink = array_pop($joinPath);
570
571 // Custom field names are already prefixed
572 $isCustom = $lastLink instanceof CustomGroupJoinable;
573 if ($isCustom) {
574 array_pop($pathArray);
575 }
576 $prefix = $pathArray ? implode('.', $pathArray) . '.' : '';
577 // Cache field info for retrieval by $this->getField()
578 foreach ($lastLink->getEntityFields() as $fieldObject) {
579 $fieldArray = $fieldObject->toArray();
580 $fieldArray['sql_name'] = '`' . $lastLink->getAlias() . '`.`' . $fieldArray['column_name'] . '`';
581 $this->addSpecField($prefix . $fieldArray['name'], $fieldArray);
582 }
583 }
584
585 /**
586 * @param string $side
587 * @param string $tableName
588 * @param string $tableAlias
589 * @param array $conditions
590 */
591 public function join($side, $tableName, $tableAlias, $conditions) {
592 // INNER JOINs take precedence over LEFT JOINs
593 if ($side != 'LEFT' || !isset($this->joins[$tableAlias])) {
594 $this->joins[$tableAlias] = $side;
595 $this->query->join($tableAlias, "$side JOIN `$tableName` `$tableAlias` ON " . implode(' AND ', $conditions));
596 }
597 }
598
599 /**
600 * @return FALSE|string
601 */
602 public function getFrom() {
603 return CoreUtil::getTableName($this->getEntity());
604 }
605
606 /**
607 * @return string
608 */
609 public function getEntity() {
610 return $this->api->getEntityName();
611 }
612
613 /**
614 * @return array
615 */
616 public function getSelect() {
617 return $this->api->getSelect();
618 }
619
620 /**
621 * @return array
622 */
623 public function getWhere() {
624 return $this->api->getWhere();
625 }
626
627 /**
628 * @return array
629 */
630 public function getHaving() {
631 return $this->api->getHaving();
632 }
633
634 /**
635 * @return array
636 */
637 public function getJoin() {
638 return $this->api->getJoin();
639 }
640
641 /**
642 * @return array
643 */
644 public function getGroupBy() {
645 return $this->api->getGroupBy();
646 }
647
648 /**
649 * @return array
650 */
651 public function getOrderBy() {
652 return $this->api->getOrderBy();
653 }
654
655 /**
656 * @return mixed
657 */
658 public function getLimit() {
659 return $this->api->getLimit();
660 }
661
662 /**
663 * @return mixed
664 */
665 public function getOffset() {
666 return $this->api->getOffset();
667 }
668
669 /**
670 * @return \CRM_Utils_SQL_Select
671 */
672 public function getQuery() {
673 return $this->query;
674 }
675
676 /**
677 * @return bool|string
678 */
679 public function getCheckPermissions() {
680 return $this->api->getCheckPermissions();
681 }
682
683 /**
684 * @param string $path
685 * @param array $field
686 */
687 private function addSpecField($path, $field) {
688 // Only add field to spec if we have permission
689 if ($this->getCheckPermissions() && !empty($field['permission']) && !\CRM_Core_Permission::check($field['permission'])) {
690 $this->apiFieldSpec[$path] = FALSE;
691 return;
692 }
693 $this->apiFieldSpec[$path] = $field;
694 }
695
696 /**
697 * Add something to the api's debug output if debugging is enabled
698 *
699 * @param $key
700 * @param $item
701 */
702 public function debug($key, $item) {
703 if ($this->api->getDebug()) {
704 $this->api->_debugOutput[$key][] = $item;
705 }
706 }
707
708 }