CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * Are we in search mode.
 218    *
 219    * @var bool
 220    */
 221   public $_search = TRUE;
 222
 223   /**
 224    * Should we skip permission checking.
 225    *
 226    * @var bool
 227    */
 228   public $_skipPermission = FALSE;
 229
 230   /**
 231    * Should we skip adding of delete clause.
 232    *
 233    * @var bool
 234    */
 235   public $_skipDeleteClause = FALSE;
 236
 237   /**
 238    * Are we in strict mode (use equality over LIKE)
 239    *
 240    * @var bool
 241    */
 242   public $_strict = FALSE;
 243
 244   /**
 245    * What operator to use to group the clauses.
 246    *
 247    * @var string
 248    */
 249   public $_operator = 'AND';
 250
 251   public $_mode = 1;
 252
 253   /**
 254    * Should we only search on primary location.
 255    *
 256    * @var bool
 257    */
 258   public $_primaryLocation = TRUE;
 259
 260   /**
 261    * Are contact ids part of the query.
 262    *
 263    * @var bool
 264    */
 265   public $_includeContactIds = FALSE;
 266
 267   /**
 268    * Should we use the smart group cache.
 269    *
 270    * @var bool
 271    */
 272   public $_smartGroupCache = TRUE;
 273
 274   /**
 275    * Should we display contacts with a specific relationship type.
 276    *
 277    * @var string
 278    */
 279   public $_displayRelationshipType;
 280
 281   /**
 282    * Reference to the query object for custom values.
 283    *
 284    * @var Object
 285    */
 286   public $_customQuery;
 287
 288   /**
 289    * Should we enable the distinct clause, used if we are including
 290    * more than one group
 291    *
 292    * @var bool
 293    */
 294   public $_useDistinct = FALSE;
 295
 296   /**
 297    * Should we just display one contact record
 298    * @var bool
 299    */
 300   public $_useGroupBy = FALSE;
 301
 302   /**
 303    * The relationship type direction
 304    *
 305    * @var array
 306    */
 307   public static $_relType;
 308
 309   /**
 310    * The activity role
 311    *
 312    * @var array
 313    */
 314   public static $_activityRole;
 315
 316   /**
 317    * Consider the component activity type
 318    * during activity search.
 319    *
 320    * @var array
 321    */
 322   public static $_considerCompActivities;
 323
 324   /**
 325    * Consider with contact activities only,
 326    * during activity search.
 327    *
 328    * @var array
 329    */
 330   public static $_withContactActivitiesOnly;
 331
 332   /**
 333    * Use distinct component clause for component searches
 334    *
 335    * @var string
 336    */
 337   public $_distinctComponentClause;
 338
 339   public $_rowCountClause;
 340
 341   /**
 342    * Use groupBy component clause for component searches
 343    *
 344    * @var string
 345    */
 346   public $_groupByComponentClause;
 347
 348   /**
 349    * Track open panes, useful in advance search
 350    *
 351    * @var array
 352    */
 353   public static $_openedPanes = [];
 354
 355   /**
 356    * For search builder - which custom fields are location-dependent
 357    * @var array
 358    */
 359   public $_locationSpecificCustomFields = [];
 360
 361   /**
 362    * The tables which have a dependency on location and/or address
 363    *
 364    * @var array
 365    */
 366   public static $_dependencies = [
 367     'civicrm_state_province' => 1,
 368     'civicrm_country' => 1,
 369     'civicrm_county' => 1,
 370     'civicrm_address' => 1,
 371     'civicrm_location_type' => 1,
 372   ];
 373
 374   /**
 375    * List of location specific fields.
 376    * @var array
 377    */
 378   public static $_locationSpecificFields = [
 379     'street_address',
 380     'street_number',
 381     'street_name',
 382     'street_unit',
 383     'supplemental_address_1',
 384     'supplemental_address_2',
 385     'supplemental_address_3',
 386     'city',
 387     'postal_code',
 388     'postal_code_suffix',
 389     'geo_code_1',
 390     'geo_code_2',
 391     'state_province',
 392     'country',
 393     'county',
 394     'phone',
 395     'email',
 396     'im',
 397     'address_name',
 398     'master_id',
 399     'location_type',
 400   ];
 401
 402   /**
 403    * Remember if we handle either end of a number or date range
 404    * so we can skip the other
 405    * @var array
 406    */
 407   protected $_rangeCache = [];
 408   /**
 409    * Set to true when $this->relationship is run to avoid adding twice.
 410    *
 411    * @var bool
 412    */
 413   protected $_relationshipValuesAdded = FALSE;
 414
 415   /**
 416    * Set to the name of the temp table if one has been created.
 417    *
 418    * @var string
 419    */
 420   public static $_relationshipTempTable;
 421
 422   public $_pseudoConstantsSelect = [];
 423
 424   public $_groupUniqueKey;
 425   public $_groupKeys = [];
 426
 427   /**
 428    * Class constructor which also does all the work.
 429    *
 430    * @param array $params
 431    * @param array $returnProperties
 432    * @param array $fields
 433    * @param bool $includeContactIds
 434    * @param bool $strict
 435    * @param bool|int $mode - mode the search is operating on
 436    *
 437    * @param bool $skipPermission
 438    * @param bool $searchDescendentGroups
 439    * @param bool $smartGroupCache
 440    * @param null $displayRelationshipType
 441    * @param string $operator
 442    * @param string $apiEntity
 443    * @param bool|null $primaryLocationOnly
 444    *
 445    * @throws \CRM_Core_Exception
 446    */
 447   public function __construct(
 448     $params = NULL, $returnProperties = NULL, $fields = NULL,
 449     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 450     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 451     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 452     $operator = 'AND',
 453     $apiEntity = NULL,
 454     $primaryLocationOnly = NULL
 455   ) {
 456     if ($primaryLocationOnly === NULL) {
 457       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 458     }
 459     $this->_primaryLocation = $primaryLocationOnly;
 460     $this->_params = &$params;
 461     if ($this->_params == NULL) {
 462       $this->_params = [];
 463     }
 464
 465     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 466       $this->_returnProperties = [];
 467     }
 468     elseif (empty($returnProperties)) {
 469       $this->_returnProperties = self::defaultReturnProperties($mode);
 470     }
 471     else {
 472       $this->_returnProperties = &$returnProperties;
 473     }
 474
 475     $this->_includeContactIds = $includeContactIds;
 476     $this->_strict = $strict;
 477     $this->_mode = $mode;
 478     $this->_skipPermission = $skipPermission;
 479     $this->_smartGroupCache = $smartGroupCache;
 480     $this->_displayRelationshipType = $displayRelationshipType;
 481     $this->setOperator($operator);
 482
 483     if ($fields) {
 484       $this->_fields = &$fields;
 485       $this->_search = FALSE;
 486       $this->_skipPermission = TRUE;
 487     }
 488     else {
 489       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 490       //  The legacy hacked fields will output as a string rather than their underlying type.
 491       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 492         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 493       }
 494       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 495       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 496       // No good option other than hard-coding metadata for this 'special' field in.
 497       $relationFields['relation_active_period_date'] = [
 498         'name' => 'relation_active_period_date',
 499         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 500         'title' => ts('Active Period'),
 501         'table_name' => 'civicrm_relationship',
 502         'where' => 'civicrm_relationship.start_date',
 503         'where_end' => 'civicrm_relationship.end_date',
 504         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 505       ];
 506       $this->_fields = array_merge($relationFields, $this->_fields);
 507
 508       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 509       unset($fields['note']);
 510       $this->_fields = array_merge($this->_fields, $fields);
 511
 512       // add activity fields
 513       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 514       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields('Case'));
 515       // Add hack as no unique name is defined for the field but the search form is in denial.
 516       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 517
 518       // add any fields provided by hook implementers
 519       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 520       $this->_fields = array_merge($this->_fields, $extFields);
 521     }
 522
 523     // basically do all the work once, and then reuse it
 524     $this->initialize($apiEntity);
 525   }
 526
 527   /**
 528    * Function which actually does all the work for the constructor.
 529    *
 530    * @param string $apiEntity
 531    *   The api entity being called.
 532    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 533    *
 534    * @throws \CRM_Core_Exception
 535    */
 536   public function initialize($apiEntity = NULL) {
 537     $this->_select = [];
 538     $this->_element = [];
 539     $this->_tables = [];
 540     $this->_whereTables = [];
 541     $this->_where = [];
 542     $this->_qill = [];
 543     $this->_cfIDs = [];
 544     $this->_paramLookup = [];
 545     $this->_having = [];
 546
 547     $this->_customQuery = NULL;
 548
 549     // reset cached static variables - CRM-5803
 550     self::$_activityRole = NULL;
 551     self::$_considerCompActivities = NULL;
 552     self::$_withContactActivitiesOnly = NULL;
 553
 554     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 555     $this->_element['contact_id'] = 1;
 556     $this->_tables['civicrm_contact'] = 1;
 557
 558     if (!empty($this->_params)) {
 559       $this->buildParamsLookup();
 560     }
 561
 562     $this->_whereTables = $this->_tables;
 563
 564     $this->selectClause($apiEntity);
 565     if (!empty($this->_cfIDs)) {
 566       // @todo This function is the select function but instead of running 'select' it
 567       // is running the whole query.
 568       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 569       $this->_customQuery->query();
 570       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 571       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 572       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 573     }
 574     $isForcePrimaryOnly = !empty($apiEntity);
 575     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 576     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 577       $component = 'contribution';
 578     }
 579     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 580       $component = 'membership';
 581     }
 582     if (isset($component) && !$this->_skipPermission) {
 583       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 584       $clauses = [];
 585       if ($component === 'contribution') {
 586         $clauses = CRM_Contribute_BAO_Contribution::getSelectWhereClause();
 587       }
 588       if ($component === 'membership') {
 589         $clauses = CRM_Member_BAO_Membership::getSelectWhereClause();
 590       }
 591       if ($clauses) {
 592         $this->_whereClause .= ' AND ' . implode(' AND ', $clauses);
 593       }
 594     }
 595
 596     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 597     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 598
 599     $this->openedSearchPanes(TRUE);
 600   }
 601
 602   /**
 603    * Function for same purpose as convertFormValues.
 604    *
 605    * Like convert form values this function exists to pre-Process parameters from the form.
 606    *
 607    * It is unclear why they are different functions & likely relates to advances search
 608    * versus search builder.
 609    *
 610    * The direction we are going is having the form convert values to a standardised format &
 611    * moving away from weird & wonderful where clause switches.
 612    *
 613    * Fix and handle contact deletion nicely.
 614    *
 615    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 616    *
 617    * @see https://issues.civicrm.org/jira/browse/CRM-11971
 618    */
 619   public function buildParamsLookup() {
 620     $trashParamExists = FALSE;
 621     $paramByGroup = [];
 622     foreach ($this->_params as $k => $param) {
 623       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 624         $trashParamExists = TRUE;
 625       }
 626       if (!empty($param[3])) {
 627         $paramByGroup[$param[3]][$k] = $param;
 628       }
 629     }
 630
 631     if ($trashParamExists) {
 632       $this->_skipDeleteClause = TRUE;
 633
 634       //cycle through group sets and explicitly add trash param if not set
 635       foreach ($paramByGroup as $setID => $set) {
 636         if (
 637           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 638           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 639         ) {
 640           $this->_params[] = [
 641             'contact_is_deleted',
 642             '=',
 643             '0',
 644             $setID,
 645             '0',
 646           ];
 647         }
 648       }
 649     }
 650
 651     foreach ($this->_params as $value) {
 652       if (empty($value[0])) {
 653         continue;
 654       }
 655       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 656       if ($cfID) {
 657         if (!array_key_exists($cfID, $this->_cfIDs)) {
 658           $this->_cfIDs[$cfID] = [];
 659         }
 660         // Set wildcard value based on "and/or" selection
 661         foreach ($this->_params as $key => $param) {
 662           if ($param[0] == $value[0] . '_operator') {
 663             $value[4] = $param[2] == 'or';
 664             break;
 665           }
 666         }
 667         $this->_cfIDs[$cfID][] = $value;
 668       }
 669
 670       if (!array_key_exists($value[0], $this->_paramLookup)) {
 671         $this->_paramLookup[$value[0]] = [];
 672       }
 673       if ($value[0] !== 'group') {
 674         // Just trying to unravel how group interacts here! This whole function is weird.
 675         $this->_paramLookup[$value[0]][] = $value;
 676       }
 677     }
 678   }
 679
 680   /**
 681    * Some composite fields do not appear in the fields array hack to make them part of the query.
 682    *
 683    * @param $apiEntity
 684    *   The api entity being called.
 685    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 686    */
 687   public function addSpecialFields($apiEntity) {
 688     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 689     // if get called via Contact.get API having address_id as return parameter
 690     if ($apiEntity === 'Contact') {
 691       $special[] = 'address_id';
 692     }
 693     foreach ($special as $name) {
 694       if (!empty($this->_returnProperties[$name])) {
 695         if ($name === 'address_id') {
 696           $this->_tables['civicrm_address'] = 1;
 697           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 698           $this->_element['address_id'] = 1;
 699         }
 700         else {
 701           $this->_select[$name] = "contact_a.{$name} as $name";
 702           $this->_element[$name] = 1;
 703         }
 704       }
 705     }
 706   }
 707
 708   /**
 709    * Given a list of conditions in params and a list of desired
 710    * return Properties generate the required select and from
 711    * clauses. Note that since the where clause introduces new
 712    * tables, the initial attempt also retrieves all variables used
 713    * in the params list
 714    *
 715    * @param string $apiEntity
 716    *   The api entity being called.
 717    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 718    */
 719   public function selectClause($apiEntity = NULL) {
 720
 721     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 722     // param ($apiEntity) instead - presumably an oversight & 2 because
 723     // contact is not implemented as a component.
 724     $this->addSpecialFields($apiEntity);
 725
 726     foreach ($this->_fields as $name => $field) {
 727       // skip component fields
 728       // there are done by the alter query below
 729       // and need not be done on every field
 730       // @todo remove these & handle using metadata - only obscure fields
 731       // that are hack-added should need to be excluded from the main loop.
 732       if (
 733         (substr($name, 0, 12) === 'participant_') ||
 734         (substr($name, 0, 7) === 'pledge_') ||
 735         (substr($name, 0, 5) === 'case_')
 736       ) {
 737         continue;
 738       }
 739
 740       // redirect to activity select clause
 741       if (
 742         (substr($name, 0, 9) === 'activity_') ||
 743         ($name === 'parent_id')
 744       ) {
 745         CRM_Activity_BAO_Query::select($this);
 746       }
 747
 748       // if this is a hierarchical name, we ignore it
 749       $names = explode('-', $name);
 750       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 751         continue;
 752       }
 753
 754       // make an exception for special cases, to add the field in select clause
 755       $makeException = FALSE;
 756
 757       //special handling for groups/tags
 758       if (in_array($name, ['groups', 'tags', 'notes'])
 759         && isset($this->_returnProperties[substr($name, 0, -1)])
 760       ) {
 761         // @todo instead of setting make exception to get us into
 762         // an if clause that has handling for these fields buried with in it
 763         // move the handling to here.
 764         $makeException = TRUE;
 765       }
 766
 767       // since note has 3 different options we need special handling
 768       // note / note_subject / note_body
 769       if ($name === 'notes') {
 770         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 771           if (isset($this->_returnProperties[$noteField])) {
 772             $makeException = TRUE;
 773             break;
 774           }
 775         }
 776       }
 777
 778       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 779       if (
 780         !empty($this->_paramLookup[$name])
 781         || !empty($this->_returnProperties[$name])
 782         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 783         || $makeException
 784       ) {
 785         if ($cfID) {
 786           // add to cfIDs array if not present
 787           if (!array_key_exists($cfID, $this->_cfIDs)) {
 788             $this->_cfIDs[$cfID] = [];
 789           }
 790         }
 791         elseif (isset($field['where'])) {
 792           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 793           if (isset($tableName)) {
 794             if (!empty(self::$_dependencies[$tableName])) {
 795               $this->_tables['civicrm_address'] = 1;
 796               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 797               $this->_element['address_id'] = 1;
 798             }
 799
 800             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 801               $tableName === 'postal_greeting' || $tableName === 'addressee'
 802             ) {
 803               if ($tableName === 'im_provider') {
 804                 CRM_Core_OptionValue::select($this);
 805               }
 806
 807               if (in_array($tableName,
 808                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 809                 $this->_element["{$name}_id"] = 1;
 810                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 811                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 812                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 813                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 814
 815                 if ($tableName === 'email_greeting') {
 816                   // @todo bad join.
 817                   $this->_pseudoConstantsSelect[$name]['join']
 818                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 819                   $this->_pseudoConstantsSelect[$name]['join'] .=
 820                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 821                 }
 822                 elseif ($tableName === 'postal_greeting') {
 823                   // @todo bad join.
 824                   $this->_pseudoConstantsSelect[$name]['join']
 825                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 826                   $this->_pseudoConstantsSelect[$name]['join'] .=
 827                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 828                 }
 829                 elseif ($tableName == 'addressee') {
 830                   // @todo bad join.
 831                   $this->_pseudoConstantsSelect[$name]['join']
 832                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 833                   $this->_pseudoConstantsSelect[$name]['join'] .=
 834                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 835                 }
 836                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 837
 838                 //get display
 839                 $greetField = "{$name}_display";
 840                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 841                 $this->_element[$greetField] = 1;
 842                 //get custom
 843                 $greetField = "{$name}_custom";
 844                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 845                 $this->_element[$greetField] = 1;
 846               }
 847             }
 848             else {
 849               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 850                 $this->_tables[$tableName] = 1;
 851               }
 852
 853               // also get the id of the tableName
 854               $tName = substr($tableName, 8);
 855               if (in_array($tName, ['country', 'state_province', 'county'])) {
 856                 if ($tName == 'state_province') {
 857                   $this->_pseudoConstantsSelect['state_province_name'] = [
 858                     'pseudoField' => "{$tName}",
 859                     'idCol' => "{$tName}_id",
 860                     'bao' => 'CRM_Core_BAO_Address',
 861                     'table' => "civicrm_{$tName}",
 862                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 863                   ];
 864
 865                   $this->_pseudoConstantsSelect[$tName] = [
 866                     'pseudoField' => 'state_province_abbreviation',
 867                     'idCol' => "{$tName}_id",
 868                     'table' => "civicrm_{$tName}",
 869                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 870                   ];
 871                 }
 872                 else {
 873                   $this->_pseudoConstantsSelect[$name] = [
 874                     'pseudoField' => "{$tName}_id",
 875                     'idCol' => "{$tName}_id",
 876                     'bao' => 'CRM_Core_BAO_Address',
 877                     'table' => "civicrm_{$tName}",
 878                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 879                   ];
 880                 }
 881
 882                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 883                 $this->_element["{$tName}_id"] = 1;
 884               }
 885               elseif ($tName != 'contact') {
 886                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 887                 $this->_element["{$tName}_id"] = 1;
 888               }
 889
 890               //special case for phone
 891               if ($name == 'phone') {
 892                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 893                 $this->_element['phone_type_id'] = 1;
 894               }
 895
 896               // if IM then select provider_id also
 897               // to get "IM Service Provider" in a file to be exported, CRM-3140
 898               if ($name == 'im') {
 899                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 900                 $this->_element['provider_id'] = 1;
 901               }
 902
 903               if ($tName == 'contact' && $fieldName == 'organization_name') {
 904                 // special case, when current employer is set for Individual contact
 905                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 906               }
 907               elseif ($tName == 'contact' && $fieldName === 'id') {
 908                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 909               }
 910               elseif (in_array($tName, ['country', 'county'])) {
 911                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 912                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 913               }
 914               elseif ($tName == 'state_province') {
 915                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 916                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 917               }
 918               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 919                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 920                   $this->_select[$name] = "{$field['where']} as `$name`";
 921                 }
 922               }
 923               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 924                 $this->addPseudoconstantFieldToSelect($name);
 925               }
 926               else {
 927                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 928               }
 929               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 930                 $this->_element[$name] = 1;
 931               }
 932             }
 933           }
 934         }
 935         elseif ($name === 'tags') {
 936           //@todo move this handling outside the big IF & ditch $makeException
 937           $this->_useGroupBy = TRUE;
 938           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 939           $this->_element[$name] = 1;
 940           $this->_tables['civicrm_tag'] = 1;
 941           $this->_tables['civicrm_entity_tag'] = 1;
 942         }
 943         elseif ($name === 'groups') {
 944           //@todo move this handling outside the big IF & ditch $makeException
 945           $this->_useGroupBy = TRUE;
 946           // Duplicates will be created here but better to sort them out in php land.
 947           $this->_select[$name] = "
 948             CONCAT_WS(',',
 949             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 950             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 951           )
 952           as `groups`";
 953           $this->_element[$name] = 1;
 954           $this->_tables['civicrm_group_contact'] = 1;
 955           $this->_tables['civicrm_group_contact_cache'] = 1;
 956           $this->_pseudoConstantsSelect["{$name}"] = [
 957             'pseudoField' => "groups",
 958             'idCol' => 'groups',
 959           ];
 960         }
 961         elseif ($name === 'notes') {
 962           //@todo move this handling outside the big IF & ditch $makeException
 963           // if note field is subject then return subject else body of the note
 964           $noteColumn = 'note';
 965           if (isset($noteField) && $noteField === 'note_subject') {
 966             $noteColumn = 'subject';
 967           }
 968
 969           $this->_useGroupBy = TRUE;
 970           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 971           $this->_element[$name] = 1;
 972           $this->_tables['civicrm_note'] = 1;
 973         }
 974         elseif ($name === 'current_employer') {
 975           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 976           $this->_element[$name] = 1;
 977         }
 978       }
 979
 980       if ($cfID && !empty($field['is_search_range'])) {
 981         // this is a custom field with range search enabled, so we better check for two/from values
 982         if (!empty($this->_paramLookup[$name . '_from'])) {
 983           if (!array_key_exists($cfID, $this->_cfIDs)) {
 984             $this->_cfIDs[$cfID] = [];
 985           }
 986           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 987             // search in the cdID array for the same grouping
 988             $fnd = FALSE;
 989             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 990               if ($c[3] == $p[3]) {
 991                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 992                 $fnd = TRUE;
 993               }
 994             }
 995             if (!$fnd) {
 996               $p[2] = ['from' => $p[2]];
 997               $this->_cfIDs[$cfID][] = $p;
 998             }
 999           }
1000         }
1001         if (!empty($this->_paramLookup[$name . '_to'])) {
1002           if (!array_key_exists($cfID, $this->_cfIDs)) {
1003             $this->_cfIDs[$cfID] = [];
1004           }
1005           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
1006             // search in the cdID array for the same grouping
1007             $fnd = FALSE;
1008             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
1009               if ($c[4] == $p[4]) {
1010                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1011                 $fnd = TRUE;
1012               }
1013             }
1014             if (!$fnd) {
1015               $p[2] = ['to' => $p[2]];
1016               $this->_cfIDs[$cfID][] = $p;
1017             }
1018           }
1019         }
1020       }
1021     }
1022
1023     // add location as hierarchical elements
1024     $this->addHierarchicalElements();
1025
1026     // add multiple field like website
1027     $this->addMultipleElements();
1028
1029     //fix for CRM-951
1030     CRM_Core_Component::alterQuery($this, 'select');
1031
1032     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1033   }
1034
1035   /**
1036    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1037    */
1038   public function addHierarchicalElements() {
1039     if (empty($this->_returnProperties['location'])) {
1040       return;
1041     }
1042     if (!is_array($this->_returnProperties['location'])) {
1043       return;
1044     }
1045
1046     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1047     $processed = [];
1048     $index = 0;
1049
1050     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1051     $addressCustomFieldIds = [];
1052
1053     foreach ($this->_returnProperties['location'] as $name => $elements) {
1054       $lCond = self::getPrimaryCondition($name);
1055       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1056
1057       if (!$lCond) {
1058         if ($locationTypeId === FALSE) {
1059           continue;
1060         }
1061         $lCond = "location_type_id = $locationTypeId";
1062         $this->_useDistinct = TRUE;
1063
1064         //commented for CRM-3256
1065         $this->_useGroupBy = TRUE;
1066       }
1067
1068       $name = str_replace(' ', '_', $name);
1069       $tName = "$name-location_type";
1070       $ltName = "`$name-location_type`";
1071       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1072       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1073       $this->_element["{$tName}_id"] = 1;
1074       $this->_element["{$tName}"] = 1;
1075
1076       $locationTypeName = $tName;
1077       $locationTypeJoin = [];
1078
1079       $addWhereCount = 0;
1080       foreach ($elements as $elementFullName => $dontCare) {
1081         $index++;
1082         $elementName = $elementCmpName = $elementFullName;
1083
1084         if (substr($elementCmpName, 0, 5) == 'phone') {
1085           $elementCmpName = 'phone';
1086         }
1087
1088         if (array_key_exists($elementCmpName, $addressCustomFields)) {
1089           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1090             $addressCustomFieldIds[$cfID][$name] = 1;
1091           }
1092         }
1093         // add address table - doesn't matter if we do it mutliple times - it's the same data
1094         // @todo ditch the double processing of addressJoin
1095         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1096           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1097         ) {
1098           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1099           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1100           $processed[$aName] = 1;
1101         }
1102
1103         $cond = $elementType = '';
1104         if (strpos($elementName, '-') !== FALSE) {
1105           // this is either phone, email or IM
1106           list($elementName, $elementType) = explode('-', $elementName);
1107
1108           if (($elementName != 'phone') && ($elementName != 'im')) {
1109             $cond = self::getPrimaryCondition($elementType);
1110           }
1111           // CRM-13011 : If location type is primary, do not restrict search to the phone
1112           // type id - we want the primary phone, regardless of what type it is.
1113           // Otherwise, restrict to the specified phone type for the given field.
1114           if ((!$cond) && ($elementName == 'phone')) {
1115             $cond = "phone_type_id = '$elementType'";
1116           }
1117           elseif ((!$cond) && ($elementName == 'im')) {
1118             // IM service provider id, CRM-3140
1119             $cond = "provider_id = '$elementType'";
1120           }
1121           $elementType = '-' . $elementType;
1122         }
1123
1124         $field = $this->_fields[$elementName] ?? NULL;
1125         if (!empty($field)) {
1126           if (isset($this->_pseudoConstantsSelect[$field['name']])) {
1127             $this->_pseudoConstantsSelect[$name . '-' . $field['name']] = $this->_pseudoConstantsSelect[$field['name']];
1128           }
1129         }
1130
1131         // hack for profile, add location id
1132         if (!$field) {
1133           if ($elementType &&
1134             // fix for CRM-882( to handle phone types )
1135             !is_numeric($elementType)
1136           ) {
1137             if (is_numeric($name)) {
1138               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1139             }
1140             else {
1141               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1142             }
1143           }
1144           elseif (is_numeric($name)) {
1145             //this for phone type to work
1146             if (in_array($elementName, ['phone', 'phone_ext'])) {
1147               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1148             }
1149             else {
1150               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1151             }
1152           }
1153           else {
1154             //this is for phone type to work for profile edit
1155             if (in_array($elementName, ['phone', 'phone_ext'])) {
1156               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1157             }
1158             else {
1159               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1160             }
1161           }
1162         }
1163
1164         // Check if there is a value, if so also add to where Clause
1165         $addWhere = FALSE;
1166         if ($this->_params) {
1167           $nm = $elementName;
1168           if (isset($locationTypeId)) {
1169             $nm .= "-$locationTypeId";
1170           }
1171           if (!is_numeric($elementType)) {
1172             $nm .= "$elementType";
1173           }
1174
1175           foreach ($this->_params as $id => $values) {
1176             if ((is_array($values) && $values[0] == $nm) ||
1177               (in_array($elementName, ['phone', 'im'])
1178                 && (strpos($values[0], $nm) !== FALSE)
1179               )
1180             ) {
1181               $addWhere = TRUE;
1182               $addWhereCount++;
1183               break;
1184             }
1185           }
1186         }
1187
1188         if ($field && isset($field['where'])) {
1189           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1190           $pf = substr($tableName, 8);
1191           $tName = $name . '-' . $pf . $elementType;
1192           if (isset($tableName)) {
1193             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1194               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1195             }
1196             else {
1197               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1198             }
1199
1200             $this->_element["{$tName}_id"] = 1;
1201             if (substr($tName, -15) == '-state_province') {
1202               // FIXME: hack to fix CRM-1900
1203               $a = Civi::settings()->get('address_format');
1204
1205               if (substr_count($a, 'state_province_name') > 0) {
1206                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1207                   'pseudoField' => "{$pf}_id",
1208                   'idCol' => "{$tName}_id",
1209                   'bao' => 'CRM_Core_BAO_Address',
1210                 ];
1211                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1212               }
1213               else {
1214                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1215                   'pseudoField' => 'state_province_abbreviation',
1216                   'idCol' => "{$tName}_id",
1217                 ];
1218                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1219               }
1220             }
1221             else {
1222               if (substr($elementFullName, 0, 2) == 'im') {
1223                 $provider = "{$name}-{$elementFullName}-provider_id";
1224                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1225                 $this->_element[$provider] = 1;
1226               }
1227               if ($pf == 'country' || $pf == 'county') {
1228                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1229                   'pseudoField' => "{$pf}_id",
1230                   'idCol' => "{$tName}_id",
1231                   'bao' => 'CRM_Core_BAO_Address',
1232                 ];
1233                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1234               }
1235               else {
1236                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1237               }
1238             }
1239
1240             if (in_array($pf, ['state_province', 'country', 'county'])) {
1241               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1242             }
1243             else {
1244               $this->_element["{$name}-{$elementFullName}"] = 1;
1245             }
1246
1247             if (empty($processed["`$tName`"])) {
1248               $processed["`$tName`"] = 1;
1249               $newName = $tableName . '_' . $index;
1250               switch ($tableName) {
1251                 case 'civicrm_phone':
1252                 case 'civicrm_email':
1253                 case 'civicrm_im':
1254                 case 'civicrm_openid':
1255
1256                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1257                   if ($tableName != 'civicrm_phone') {
1258                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1259                   }
1260                   elseif (is_numeric($name)) {
1261                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1262                   }
1263
1264                   // this special case to add phone type
1265                   if ($cond) {
1266                     $phoneTypeCondition = " AND `$tName`.$cond ";
1267                     //gross hack to pickup corrupted data also, CRM-7603
1268                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1269                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1270                       if (!empty($lCond)) {
1271                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1272                       }
1273                     }
1274                     $this->_tables[$tName] .= $phoneTypeCondition;
1275                   }
1276
1277                   //build locationType join
1278                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1279
1280                   if ($addWhere) {
1281                     $this->_whereTables[$tName] = $this->_tables[$tName];
1282                   }
1283                   break;
1284
1285                 case 'civicrm_state_province':
1286                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1287                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1288                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1289                   if ($addWhere) {
1290                     $this->_whereTables["{$name}-address"] = $addressJoin;
1291                   }
1292                   break;
1293
1294                 case 'civicrm_location_type':
1295                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.location_type_id";
1296
1297                   if ($addWhere) {
1298                     $this->_whereTables["{$name}-address"] = $addressJoin;
1299                   }
1300                   break;
1301
1302                 case 'civicrm_country':
1303                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1304                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1305                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1306                   if ($addWhere) {
1307                     $this->_whereTables["{$name}-address"] = $addressJoin;
1308                   }
1309                   break;
1310
1311                 case 'civicrm_county':
1312                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1313                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1314                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1315                   if ($addWhere) {
1316                     $this->_whereTables["{$name}-address"] = $addressJoin;
1317                   }
1318                   break;
1319
1320                 default:
1321                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1322                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1323                   }
1324                   if ($addWhere) {
1325                     $this->_whereTables["{$name}-address"] = $addressJoin;
1326                   }
1327                   break;
1328               }
1329             }
1330           }
1331         }
1332       }
1333
1334       // add location type  join
1335       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1336       $this->_tables[$locationTypeName] = $ltypeJoin;
1337
1338       // table should be present in $this->_whereTables,
1339       // to add its condition in location type join, CRM-3939.
1340       if ($addWhereCount) {
1341         $locClause = [];
1342         foreach ($this->_whereTables as $tableName => $clause) {
1343           if (!empty($locationTypeJoin[$tableName])) {
1344             $locClause[] = $locationTypeJoin[$tableName];
1345           }
1346         }
1347
1348         if (!empty($locClause)) {
1349           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1350         }
1351       }
1352     }
1353
1354     if (!empty($addressCustomFieldIds)) {
1355       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1356       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1357         foreach ($locTypeName as $name => $dnc) {
1358           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1359           $fieldName = "$name-custom_{$cfID}";
1360           $tName = "$name-address-custom-{$cfID}";
1361           $aName = "`$name-address-custom-{$cfID}`";
1362           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1363           $this->_element["{$tName}_id"] = 1;
1364           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1365           $this->_element[$fieldName] = 1;
1366           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1367         }
1368       }
1369     }
1370   }
1371
1372   /**
1373    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1374    */
1375   public function addMultipleElements() {
1376     if (empty($this->_returnProperties['website'])) {
1377       return;
1378     }
1379     if (!is_array($this->_returnProperties['website'])) {
1380       return;
1381     }
1382
1383     foreach ($this->_returnProperties['website'] as $key => $elements) {
1384       foreach ($elements as $elementFullName => $dontCare) {
1385         $tName = "website-{$key}-{$elementFullName}";
1386         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1387         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1388         $this->_element["{$tName}_id"] = 1;
1389         $this->_element["{$tName}"] = 1;
1390
1391         $type = "website-{$key}-website_type_id";
1392         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1393         $this->_element[$type] = 1;
1394         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1395       }
1396     }
1397   }
1398
1399   /**
1400    * Generate the query based on what type of query we need.
1401    *
1402    * @param bool $count
1403    * @param bool $sortByChar
1404    * @param bool $groupContacts
1405    * @param bool $onlyDeleted
1406    *
1407    * @return array
1408    *   sql query parts as an array
1409    */
1410   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1411     // build permission clause
1412     $this->generatePermissionClause($onlyDeleted, $count);
1413
1414     if ($count) {
1415       if (isset($this->_rowCountClause)) {
1416         $select = "SELECT {$this->_rowCountClause}";
1417       }
1418       elseif (isset($this->_distinctComponentClause)) {
1419         // we add distinct to get the right count for components
1420         // for the more complex result set, we use GROUP BY the same id
1421         // CRM-9630
1422         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1423       }
1424       else {
1425         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1426       }
1427       $from = $this->_simpleFromClause;
1428       if ($this->_useDistinct) {
1429         $this->_useGroupBy = TRUE;
1430       }
1431     }
1432     elseif ($sortByChar) {
1433       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1434       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1435       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1436       $from = $this->_simpleFromClause;
1437     }
1438     elseif ($groupContacts) {
1439       $select = 'SELECT contact_a.id as id';
1440       if ($this->_useDistinct) {
1441         $this->_useGroupBy = TRUE;
1442       }
1443       $from = $this->_simpleFromClause;
1444     }
1445     else {
1446       if (!empty($this->_paramLookup['group'])) {
1447
1448         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1449
1450         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1451           $this->_paramLookup['group'][0][1] = key($value);
1452         }
1453
1454         // Presumably the lines below come into manage groups screen.
1455         // make sure there is only one element
1456         // this is used when we are running under smog and need to know
1457         // how the contact was added (CRM-1203)
1458         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1459         if ((count($this->_paramLookup['group']) == 1) &&
1460           (count($groups) == 1)
1461         ) {
1462           $groupId = $groups[0];
1463
1464           //check if group is saved search
1465           $group = new CRM_Contact_BAO_Group();
1466           $group->id = $groupId;
1467           $group->find(TRUE);
1468
1469           if (!isset($group->saved_search_id)) {
1470             $tbName = "civicrm_group_contact";
1471             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1472             // as this will add load to an intentionally light query.
1473             // ideally this code would be removed as it appears to be to support CRM-1203
1474             // and passing in the required returnProperties from the url would
1475             // make more sense that globally applying the requirements of one form.
1476             if (($this->_returnProperties != ['contact_id'])) {
1477               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1478               $this->_element['group_contact_id'] = 1;
1479               $this->_select['status'] = "$tbName.status as status";
1480               $this->_element['status'] = 1;
1481             }
1482           }
1483         }
1484         $this->_useGroupBy = TRUE;
1485       }
1486       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1487         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1488           // CRM-5954
1489           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1490           $this->_useDistinct = FALSE;
1491           $this->_useGroupBy = TRUE;
1492         }
1493       }
1494
1495       $select = $this->getSelect();
1496       $from = $this->_fromClause;
1497     }
1498
1499     $where = '';
1500     if (!empty($this->_whereClause)) {
1501       $where = "WHERE {$this->_whereClause}";
1502     }
1503
1504     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1505       if (!empty($this->_permissionFromClause)) {
1506         $from .= " $this->_permissionFromClause";
1507       }
1508       if (empty($where)) {
1509         $where = "WHERE $this->_permissionWhereClause";
1510       }
1511       else {
1512         $where = "$where AND $this->_permissionWhereClause";
1513       }
1514     }
1515
1516     $having = '';
1517     if (!empty($this->_having)) {
1518       foreach ($this->_having as $havingSets) {
1519         foreach ($havingSets as $havingSet) {
1520           $havingValue[] = $havingSet;
1521         }
1522       }
1523       $having = ' HAVING ' . implode(' AND ', $havingValue);
1524     }
1525
1526     // if we are doing a transform, do it here
1527     // use the $from, $where and $having to get the contact ID
1528     if ($this->_displayRelationshipType) {
1529       $this->filterRelatedContacts($from, $where, $having);
1530     }
1531
1532     return [$select, $from, $where, $having];
1533   }
1534
1535   /**
1536    * Get where values from the parameters.
1537    *
1538    * @param string $name
1539    * @param mixed $grouping
1540    *
1541    * @return mixed
1542    */
1543   public function getWhereValues($name, $grouping) {
1544     $result = NULL;
1545     foreach ($this->_params as $values) {
1546       if ($values[0] == $name && $values[3] == $grouping) {
1547         return $values;
1548       }
1549     }
1550
1551     return $result;
1552   }
1553
1554   /**
1555    * Fix date values.
1556    *
1557    * @param bool $relative
1558    * @param string $from
1559    * @param string $to
1560    */
1561   public static function fixDateValues($relative, &$from, &$to) {
1562     if ($relative) {
1563       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1564     }
1565   }
1566
1567   /**
1568    * Convert values from form-appropriate to query-object appropriate.
1569    *
1570    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1571    * So, ideally we would convert all fields to look like
1572    *  array(
1573    *   0 => $fieldName
1574    *   // Set the operator for legacy reasons, but it is ignored
1575    *   1 =>  '='
1576    *   // array in sql filter syntax
1577    *   2 => array('BETWEEN' => array(1,60),
1578    *   3 => null
1579    *   4 => null
1580    *  );
1581    *
1582    * There are some examples of the syntax in
1583    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1584    *
1585    * More notes at CRM_Core_DAO::createSQLFilter
1586    *
1587    * and a list of supported operators in CRM_Core_DAO
1588    *
1589    * @param array $formValues
1590    * @param int $wildcard
1591    * @param bool $useEquals
1592    *
1593    * @param string $apiEntity
1594    *
1595    * @param array $entityReferenceFields
1596    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1597    *
1598    * @return array
1599    */
1600   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1601     $entityReferenceFields = []) {
1602     $params = [];
1603     if (empty($formValues)) {
1604       return $params;
1605     }
1606
1607     self::filterCountryFromValuesIfStateExists($formValues);
1608     CRM_Core_BAO_CustomValue::fixCustomFieldValue($formValues);
1609
1610     foreach ($formValues as $id => $values) {
1611       if (self::isAlreadyProcessedForQueryFormat($values)) {
1612         $params[] = $values;
1613         continue;
1614       }
1615
1616       self::legacyConvertFormValues($id, $values);
1617
1618       // The form uses 1 field to represent two db fields
1619       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1620         $contactType = [];
1621         $subType = [];
1622         foreach ((array) $values as $key => $type) {
1623           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1624           $contactType[$types[0]] = $types[0];
1625           // Add sub-type if specified
1626           if (!empty($types[1])) {
1627             $subType[$types[1]] = $types[1];
1628           }
1629         }
1630         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1631         if ($subType) {
1632           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1633         }
1634       }
1635       elseif ($id === 'privacy') {
1636         if (is_array($formValues['privacy'])) {
1637           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1638           foreach ($formValues['privacy'] as $key => $value) {
1639             if ($value) {
1640               $params[] = [$key, $op, $value, 0, 0];
1641             }
1642           }
1643         }
1644       }
1645       elseif ($id === 'email_on_hold') {
1646         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1647           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1648           // so in 5.11 we have an extra if that should become redundant over time.
1649           // https://lab.civicrm.org/dev/core/issues/745
1650           // @todo this renaming of email_on_hold to on_hold needs revisiting
1651           // it precedes recent changes but causes the default not to reload.
1652           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1653           if (!empty($onHoldValue)) {
1654             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1655           }
1656         }
1657       }
1658       elseif (substr($id, 0, 7) === 'custom_'
1659         &&  (
1660           substr($id, -5, 5) === '_from'
1661           || substr($id, -3, 3) === '_to'
1662         )
1663       ) {
1664         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1665       }
1666       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1667         FALSE)) {
1668         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1669       }
1670       else {
1671         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1672
1673         if (!$values) {
1674           continue;
1675         }
1676         $params[] = $values;
1677       }
1678     }
1679     return $params;
1680   }
1681
1682   /**
1683    * Function to support legacy format for groups and tags.
1684    *
1685    * @param string $id
1686    * @param array|int $values
1687    *
1688    */
1689   public static function legacyConvertFormValues($id, &$values) {
1690     $legacyElements = [
1691       'group',
1692       'tag',
1693       'contact_tags',
1694       'contact_type',
1695       'membership_type_id',
1696       'membership_status_id',
1697     ];
1698     if (in_array($id, $legacyElements) && is_array($values)) {
1699       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1700       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1701       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1702     }
1703   }
1704
1705   /**
1706    * Fix values from query from/to something no-one cared enough to document.
1707    *
1708    * @param int $id
1709    * @param array $values
1710    * @param int $wildcard
1711    * @param bool $useEquals
1712    *
1713    * @param string $apiEntity
1714    *
1715    * @return array|null
1716    */
1717   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1718     // skip a few search variables
1719     static $skipWhere = NULL;
1720     static $likeNames = NULL;
1721     $result = NULL;
1722
1723     // Change camelCase EntityName to lowercase with underscores
1724     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1725
1726     // check if $value is in OK (Operator as Key) format as used by Get API
1727     if (CRM_Utils_System::isNull($values)) {
1728       return $result;
1729     }
1730
1731     if (!$skipWhere) {
1732       $skipWhere = [
1733         'task',
1734         'radio_ts',
1735         'uf_group_id',
1736         'component_mode',
1737         'qfKey',
1738         'operator',
1739         'display_relationship_type',
1740       ];
1741     }
1742
1743     if (in_array($id, $skipWhere) ||
1744       substr($id, 0, 4) == '_qf_' ||
1745       substr($id, 0, 7) == 'hidden_'
1746     ) {
1747       return $result;
1748     }
1749
1750     if ($apiEntity &&
1751       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1752       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1753       (substr($id, 0, 7) != 'custom_')
1754     ) {
1755       $id = $apiEntity . '_' . $id;
1756     }
1757
1758     if (!$likeNames) {
1759       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1760     }
1761
1762     // email comes in via advanced search
1763     // so use wildcard always
1764     if ($id == 'email') {
1765       $wildcard = 1;
1766     }
1767
1768     if (!$useEquals && in_array($id, $likeNames)) {
1769       $result = [$id, 'LIKE', $values, 0, 1];
1770     }
1771     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1772       $result = [$id, 'LIKE', $values, 0, 0];
1773     }
1774     elseif ($id == 'contact_type' ||
1775       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1776     ) {
1777       $result = [$id, 'IN', $values, 0, $wildcard];
1778     }
1779     else {
1780       $result = [$id, '=', $values, 0, $wildcard];
1781     }
1782
1783     return $result;
1784   }
1785
1786   /**
1787    * Get the where clause for a single field.
1788    *
1789    * @param array $values
1790    * @param bool $isForcePrimaryOnly
1791    *
1792    * @throws \CRM_Core_Exception
1793    */
1794   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1795     if ($this->isARelativeDateField($values[0])) {
1796       $this->buildRelativeDateQuery($values);
1797       return;
1798     }
1799     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1800
1801     // do not process custom fields or prefixed contact ids or component params
1802     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1803       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1804       (substr($values[0], 0, 13) === 'contribution_') ||
1805       (substr($values[0], 0, 6) === 'event_') ||
1806       (substr($values[0], 0, 12) === 'participant_') ||
1807       (substr($values[0], 0, 7) === 'member_') ||
1808       (substr($values[0], 0, 6) === 'grant_') ||
1809       (substr($values[0], 0, 7) === 'pledge_') ||
1810       (substr($values[0], 0, 5) === 'case_') ||
1811       (substr($values[0], 0, 10) === 'financial_') ||
1812       (substr($values[0], 0, 8) === 'payment_') ||
1813       (substr($values[0], 0, 11) === 'membership_')
1814       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1815       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1816       // be needed.
1817       || $values[0] === 'mailing_id'
1818     ) {
1819       return;
1820     }
1821
1822     // skip for hook injected fields / params
1823     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1824     if (array_key_exists($values[0], $extFields)) {
1825       return;
1826     }
1827
1828     switch ($values[0]) {
1829       case 'deleted_contacts':
1830         $this->deletedContacts($values);
1831         return;
1832
1833       case 'contact_sub_type':
1834         $this->contactSubType($values);
1835         return;
1836
1837       case 'group':
1838       case 'group_type':
1839         $this->group($values);
1840         return;
1841
1842       // case tag comes from find contacts
1843       case 'tag_search':
1844         $this->tagSearch($values);
1845         return;
1846
1847       case 'tag':
1848       case 'contact_tags':
1849         $this->tag($values);
1850         return;
1851
1852       case 'note':
1853       case 'note_body':
1854       case 'note_subject':
1855         $this->notes($values);
1856         return;
1857
1858       case 'uf_user':
1859         $this->ufUser($values);
1860         return;
1861
1862       case 'sort_name':
1863       case 'display_name':
1864         $this->sortName($values);
1865         return;
1866
1867       case 'addressee':
1868       case 'postal_greeting':
1869       case 'email_greeting':
1870         $this->greetings($values);
1871         return;
1872
1873       case 'email':
1874       case 'email_id':
1875         $this->email($values, $isForcePrimaryOnly);
1876         return;
1877
1878       case 'phone_numeric':
1879         $this->phone_numeric($values);
1880         return;
1881
1882       case 'phone_phone_type_id':
1883       case 'phone_location_type_id':
1884         $this->phone_option_group($values);
1885         return;
1886
1887       case 'street_address':
1888         $this->street_address($values);
1889         return;
1890
1891       case 'street_number':
1892         $this->street_number($values);
1893         return;
1894
1895       case 'sortByCharacter':
1896         $this->sortByCharacter($values);
1897         return;
1898
1899       case 'location_type':
1900         $this->locationType($values);
1901         return;
1902
1903       case 'county':
1904         $this->county($values);
1905         return;
1906
1907       case 'state_province':
1908       case 'state_province_id':
1909       case 'state_province_name':
1910         $this->stateProvince($values);
1911         return;
1912
1913       case 'country':
1914       case 'country_id':
1915         $this->country($values, FALSE);
1916         return;
1917
1918       case 'postal_code':
1919       case 'postal_code_low':
1920       case 'postal_code_high':
1921         $this->postalCode($values);
1922         return;
1923
1924       case 'activity_date':
1925       case 'activity_date_low':
1926       case 'activity_date_high':
1927       case 'activity_date_time_low':
1928       case 'activity_date_time_high':
1929       case 'activity_role':
1930       case 'activity_status_id':
1931       case 'activity_status':
1932       case 'activity_priority':
1933       case 'activity_priority_id':
1934       case 'followup_parent_id':
1935       case 'parent_id':
1936       case 'source_contact_id':
1937       case 'activity_text':
1938       case 'activity_option':
1939       case 'test_activities':
1940       case 'activity_type_id':
1941       case 'activity_type':
1942       case 'activity_survey_id':
1943       case 'activity_tags':
1944       case 'activity_taglist':
1945       case 'activity_test':
1946       case 'activity_campaign_id':
1947       case 'activity_engagement_level':
1948       case 'activity_id':
1949       case 'activity_result':
1950       case 'source_contact':
1951         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1952         return;
1953
1954       case 'age_low':
1955       case 'age_high':
1956       case 'birth_date_low':
1957       case 'birth_date_high':
1958       case 'deceased_date_low':
1959       case 'deceased_date_high':
1960         $this->demographics($values);
1961         return;
1962
1963       case 'age_asof_date':
1964         // handled by demographics
1965         return;
1966
1967       case 'log_date_low':
1968       case 'log_date_high':
1969         $this->modifiedDates($values);
1970         return;
1971
1972       case 'changed_by':
1973         $this->changeLog($values);
1974         return;
1975
1976       case 'do_not_phone':
1977       case 'do_not_email':
1978       case 'do_not_mail':
1979       case 'do_not_sms':
1980       case 'do_not_trade':
1981       case 'is_opt_out':
1982         $this->privacy($values);
1983         return;
1984
1985       case 'privacy_options':
1986         $this->privacyOptions($values);
1987         return;
1988
1989       case 'privacy_operator':
1990       case 'privacy_toggle':
1991         // these are handled by privacy options
1992         return;
1993
1994       case 'preferred_communication_method':
1995         $this->preferredCommunication($values);
1996         return;
1997
1998       case 'relation_type_id':
1999       case 'relationship_start_date_high':
2000       case 'relationship_start_date_low':
2001       case 'relationship_end_date_high':
2002       case 'relationship_end_date_low':
2003       case 'relation_active_period_date_high':
2004       case 'relation_active_period_date_low':
2005       case 'relation_target_name':
2006       case 'relation_status':
2007       case 'relation_description':
2008       case 'relation_date_low':
2009       case 'relation_date_high':
2010         $this->relationship($values);
2011         $this->_relationshipValuesAdded = TRUE;
2012         return;
2013
2014       case 'task_status_id':
2015         $this->task($values);
2016         return;
2017
2018       case 'task_id':
2019         // since this case is handled with the above
2020         return;
2021
2022       case 'prox_distance':
2023         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2024         return;
2025
2026       case 'prox_street_address':
2027       case 'prox_city':
2028       case 'prox_postal_code':
2029       case 'prox_state_province_id':
2030       case 'prox_country_id':
2031       case 'prox_geo_code_1':
2032       case 'prox_geo_code_2':
2033         // handled by the proximity_distance clause
2034         return;
2035
2036       default:
2037         $this->restWhere($values);
2038         return;
2039     }
2040   }
2041
2042   /**
2043    * Given a list of conditions in params generate the required where clause.
2044    *
2045    * @param bool $isForcePrimaryEmailOnly
2046    *
2047    * @return string
2048    * @throws \CRM_Core_Exception
2049    */
2050   public function whereClause($isForcePrimaryEmailOnly = NULL): string {
2051     $this->_where[0] = [];
2052     $this->_qill[0] = [];
2053
2054     $this->includeContactIDs();
2055     if (!empty($this->_params)) {
2056       foreach (array_keys($this->_params) as $id) {
2057         if (empty($this->_params[$id][0])) {
2058           continue;
2059         }
2060         // check for both id and contact_id
2061         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2062           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2063           $field = $this->_fields['id'] ?? NULL;
2064           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2065             'CRM_Contact_BAO_Contact',
2066             "contact_a.id",
2067             $this->_params[$id][2],
2068             $this->_params[$id][1]
2069           );
2070           $this->_qill[0][] = ts("%1 %2 %3", [
2071             1 => $field['title'] ?? '',
2072             2 => $qillop,
2073             3 => $qillVal,
2074           ]);
2075         }
2076         else {
2077           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2078         }
2079       }
2080
2081       CRM_Core_Component::alterQuery($this, 'where');
2082
2083       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2084     }
2085
2086     if ($this->_customQuery) {
2087       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2088       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2089       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2090       if (!empty($this->_customQuery->_where)) {
2091         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2092       }
2093       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2094     }
2095
2096     $clauses = [];
2097     $andClauses = [];
2098
2099     $validClauses = 0;
2100     if (!empty($this->_where)) {
2101       foreach ($this->_where as $grouping => $values) {
2102         if ($grouping > 0 && !empty($values)) {
2103           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2104           $validClauses++;
2105         }
2106       }
2107
2108       if (!empty($this->_where[0])) {
2109         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2110       }
2111       if (!empty($clauses)) {
2112         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2113       }
2114
2115       if ($validClauses > 1) {
2116         $this->_useDistinct = TRUE;
2117       }
2118     }
2119
2120     return $andClauses ? implode(' AND ', $andClauses) : ' 1 ';
2121   }
2122
2123   /**
2124    * Generate where clause for any parameters not already handled.
2125    *
2126    * @param array $values
2127    *
2128    * @throws Exception
2129    */
2130   public function restWhere(&$values) {
2131     $name = $values[0] ?? NULL;
2132     $op = $values[1] ?? NULL;
2133     $value = $values[2] ?? NULL;
2134     $grouping = $values[3] ?? NULL;
2135     $wildcard = $values[4] ?? NULL;
2136
2137     if (isset($grouping) && empty($this->_where[$grouping])) {
2138       $this->_where[$grouping] = [];
2139     }
2140
2141     $multipleFields = ['url'];
2142
2143     //check if the location type exists for fields
2144     $lType = '';
2145     $locType = explode('-', $name);
2146
2147     if (!in_array($locType[0], $multipleFields)) {
2148       //add phone type if exists
2149       if (isset($locType[2]) && $locType[2]) {
2150         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2151       }
2152     }
2153
2154     $field = $this->_fields[$name] ?? NULL;
2155
2156     if (!$field) {
2157       $field = $this->_fields[$locType[0]] ?? NULL;
2158
2159       if (!$field) {
2160         // Strip any trailing _high & _low that might be appended.
2161         $realFieldName = str_replace(['_high', '_low'], '', $name);
2162         if (isset($this->_fields[$realFieldName])) {
2163           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2164           $columnName = $field['column_name'] ?? $field['name'];
2165           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2166         }
2167         return;
2168       }
2169     }
2170
2171     $setTables = TRUE;
2172
2173     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2174     if (isset($locType[1]) && is_numeric($locType[1])) {
2175       $lType = $locationType[$locType[1]];
2176     }
2177     if ($lType) {
2178       $field['title'] .= " ($lType)";
2179     }
2180
2181     if (substr($name, 0, 14) === 'state_province') {
2182       if (isset($locType[1]) && is_numeric($locType[1])) {
2183         $setTables = FALSE;
2184         $aName = "{$lType}-address";
2185         $where = "`$aName`.state_province_id";
2186       }
2187       else {
2188         $where = "civicrm_address.state_province_id";
2189       }
2190
2191       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2192       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2193       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2194       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2195     }
2196     elseif (!empty($field['pseudoconstant'])) {
2197       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2198       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2199       $this->optionValueQuery(
2200         $name, $op, $value, $grouping,
2201         'CRM_Contact_DAO_Contact',
2202         $field,
2203         $field['html']['label'] ?? $field['title'],
2204         CRM_Utils_Type::typeToString($dataType)
2205       );
2206       if ($name === 'gender_id') {
2207         self::$_openedPanes[ts('Demographics')] = TRUE;
2208       }
2209     }
2210     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2211       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2212       if (isset($locType[1]) && is_numeric($locType[1])) {
2213         $setTables = FALSE;
2214         $aName = "{$lType}-address";
2215         $where = "`$aName`.$name";
2216       }
2217       else {
2218         $where = "civicrm_address.$name";
2219       }
2220
2221       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2222       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2223
2224       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2225       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2226     }
2227     elseif ($name === 'world_region') {
2228       $this->optionValueQuery(
2229         $name, $op, $value, $grouping,
2230         NULL,
2231         $field,
2232         ts('World Region'),
2233         'Positive'
2234       );
2235     }
2236     elseif ($name === 'is_deceased') {
2237       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2238       self::$_openedPanes[ts('Demographics')] = TRUE;
2239     }
2240     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2241       $appendDateTime = TRUE;
2242       if ($name === 'deceased_date' || $name === 'birth_date') {
2243         $appendDateTime = FALSE;
2244         self::$_openedPanes[ts('Demographics')] = TRUE;
2245       }
2246       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2247     }
2248     elseif ($name === 'contact_id') {
2249       if (is_int($value)) {
2250         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2251         $this->_qill[$grouping][] = "$field[title] $op $value";
2252       }
2253     }
2254     elseif ($name === 'name') {
2255       $value = CRM_Core_DAO::escapeString($value);
2256       if ($wildcard) {
2257         $op = 'LIKE';
2258         $value = self::getWildCardedValue($wildcard, $op, $value);
2259       }
2260       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2261       // @todo it's likely this code path is obsolete / never called. It is definitely not
2262       // passed through in our test suite.
2263       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2264       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2265     }
2266     elseif ($name === 'current_employer') {
2267       if ($wildcard) {
2268         $op = 'LIKE';
2269         $value = self::getWildCardedValue($wildcard, $op, $value);
2270       }
2271       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2272         $value
2273       );
2274       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2275       $this->_where[$grouping][] = $ceWhereClause;
2276       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2277     }
2278     elseif (substr($name, 0, 4) === 'url-') {
2279       $tName = 'civicrm_website';
2280       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2281       $value = CRM_Core_DAO::escapeString($value);
2282       if ($wildcard) {
2283         $op = 'LIKE';
2284         $value = self::getWildCardedValue($wildcard, $op, $value);
2285       }
2286
2287       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2288       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2289     }
2290     elseif ($name === 'contact_is_deleted') {
2291       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2292     }
2293     elseif (!empty($field['where'])) {
2294       $type = NULL;
2295       if (!empty($field['type'])) {
2296         $type = CRM_Utils_Type::typeToString($field['type']);
2297       }
2298
2299       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2300
2301       if (isset($locType[1]) &&
2302         is_numeric($locType[1])
2303       ) {
2304         $setTables = FALSE;
2305
2306         //get the location name
2307         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2308         $fieldName = "`$tName`.$fldName";
2309
2310         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2311         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2312
2313       }
2314       else {
2315         if ($tableName == 'civicrm_contact') {
2316           $fieldName = "contact_a.{$fieldName}";
2317         }
2318         else {
2319           $fieldName = $field['where'];
2320         }
2321       }
2322
2323       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2324       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2325         1 => $field['title'],
2326         2 => $qillop,
2327         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2328       ]);
2329
2330       if (is_array($value)) {
2331         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2332         // for api style operators for functions that hit this point without worrying about regression
2333         // (the previous comments indicated the condition for hitting this point were unknown
2334         // per CRM-14743 we are adding modified_date & created_date operator support
2335         $operations = array_keys($value);
2336         foreach ($operations as $operator) {
2337           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2338             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2339             //so this condition will satisfy the search for now
2340             if (strpos($op, 'IN') !== FALSE) {
2341               $value = [$op => $value];
2342             }
2343             // we don't know when this might happen
2344             else {
2345               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2346             }
2347           }
2348         }
2349         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2350       }
2351       else {
2352         if ($wildcard) {
2353           $op = 'LIKE';
2354           $value = self::getWildCardedValue($wildcard, $op, $value);
2355         }
2356
2357         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2358       }
2359     }
2360
2361     if ($setTables && isset($field['where'])) {
2362       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2363       if (isset($tableName)) {
2364         $this->_tables[$tableName] = 1;
2365         $this->_whereTables[$tableName] = 1;
2366       }
2367     }
2368   }
2369
2370   /**
2371    * @param $where
2372    * @param $locType
2373    *
2374    * @return array
2375    * @throws Exception
2376    */
2377   public static function getLocationTableName(&$where, &$locType) {
2378     if (isset($locType[1]) && is_numeric($locType[1])) {
2379       list($tbName, $fldName) = explode(".", $where);
2380
2381       //get the location name
2382       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2383       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2384       if (in_array($locType[0], $specialFields)) {
2385         //hack to fix / special handing for phone_ext
2386         if ($locType[0] == 'phone_ext') {
2387           $locType[0] = 'phone';
2388         }
2389         if (isset($locType[2]) && $locType[2]) {
2390           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2391         }
2392         else {
2393           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2394         }
2395       }
2396       elseif (in_array($locType[0],
2397         [
2398           'address_name',
2399           'street_address',
2400           'street_name',
2401           'street_number_suffix',
2402           'street_unit',
2403           'supplemental_address_1',
2404           'supplemental_address_2',
2405           'supplemental_address_3',
2406           'city',
2407           'postal_code',
2408           'postal_code_suffix',
2409           'geo_code_1',
2410           'geo_code_2',
2411           'master_id',
2412         ]
2413       )) {
2414         //fix for search by profile with address fields.
2415         $tName = "{$locationType[$locType[1]]}-address";
2416       }
2417       elseif (in_array($locType[0],
2418           [
2419             'on_hold',
2420             'signature_html',
2421             'signature_text',
2422             'is_bulkmail',
2423           ]
2424         )) {
2425         $tName = "{$locationType[$locType[1]]}-email";
2426       }
2427       elseif ($locType[0] == 'provider_id') {
2428         $tName = "{$locationType[$locType[1]]}-im";
2429       }
2430       elseif ($locType[0] == 'openid') {
2431         $tName = "{$locationType[$locType[1]]}-openid";
2432       }
2433       else {
2434         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2435       }
2436       $tName = str_replace(' ', '_', $tName);
2437       return [$tName, $fldName];
2438     }
2439     throw new CRM_Core_Exception('Cannot determine location table information');
2440   }
2441
2442   /**
2443    * Given a result dao, extract the values and return that array
2444    *
2445    * @param CRM_Core_DAO $dao
2446    *
2447    * @return array
2448    *   values for this query
2449    */
2450   public function store($dao) {
2451     $value = [];
2452
2453     foreach ($this->_element as $key => $dontCare) {
2454       if (property_exists($dao, $key)) {
2455         if (strpos($key, '-') !== FALSE) {
2456           $values = explode('-', $key);
2457           $lastElement = array_pop($values);
2458           $current = &$value;
2459           $cnt = count($values);
2460           $count = 1;
2461           foreach ($values as $v) {
2462             if (!array_key_exists($v, $current)) {
2463               $current[$v] = [];
2464             }
2465             //bad hack for im_provider
2466             if ($lastElement == 'provider_id') {
2467               if ($count < $cnt) {
2468                 $current = &$current[$v];
2469               }
2470               else {
2471                 $lastElement = "{$v}_{$lastElement}";
2472               }
2473             }
2474             else {
2475               $current = &$current[$v];
2476             }
2477             $count++;
2478           }
2479
2480           $current[$lastElement] = $dao->$key;
2481         }
2482         else {
2483           $value[$key] = $dao->$key;
2484         }
2485       }
2486     }
2487     return $value;
2488   }
2489
2490   /**
2491    * Getter for tables array.
2492    *
2493    * @return array
2494    */
2495   public function tables() {
2496     return $this->_tables;
2497   }
2498
2499   /**
2500    * Sometimes used to create the from clause, but, not reliably, set
2501    * this AND set tables.
2502    *
2503    * It's unclear the intent - there is a 'simpleFrom' clause which
2504    * takes whereTables into account & a fromClause which doesn't.
2505    *
2506    * logic may have eroded?
2507    *
2508    * @return array
2509    */
2510   public function whereTables() {
2511     return $this->_whereTables;
2512   }
2513
2514   /**
2515    * Generate the where clause (used in match contacts and permissions)
2516    *
2517    * @param array $params
2518    * @param array $fields
2519    * @param array $tables
2520    * @param $whereTables
2521    * @param bool $strict
2522    *
2523    * @return string
2524    * @throws \CRM_Core_Exception
2525    */
2526   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2527     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2528       FALSE, $strict
2529     );
2530
2531     $tables = array_merge($query->tables(), $tables);
2532     $whereTables = array_merge($query->whereTables(), $whereTables);
2533
2534     return $query->_whereClause;
2535   }
2536
2537   /**
2538    * Create the from clause.
2539    *
2540    * @param array $tables
2541    *   Tables that need to be included in this from clause. If null,
2542    *   return mimimal from clause (i.e. civicrm_contact).
2543    * @param array $inner
2544    *   Tables that should be inner-joined.
2545    * @param array $right
2546    *   Tables that should be right-joined.
2547    * @param bool $primaryLocation
2548    *   Search on primary location. See note below.
2549    * @param int $mode
2550    *   Determines search mode based on bitwise MODE_* constants.
2551    * @param string|null $apiEntity
2552    *   Determines search mode based on entity by string.
2553    *
2554    * The $primaryLocation flag only seems to be used when
2555    * locationType() has been called. This may be a search option
2556    * exposed, or perhaps it's a "search all details" approach which
2557    * predates decoupling of location types and primary fields?
2558    *
2559    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2560    *
2561    * @return string
2562    *   the from clause
2563    */
2564   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2565
2566     $from = ' FROM civicrm_contact contact_a';
2567     if (empty($tables)) {
2568       return $from;
2569     }
2570
2571     if (!empty($tables['civicrm_worldregion'])) {
2572       $tables = array_merge(['civicrm_country' => 1], $tables);
2573     }
2574
2575     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2576         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2577       $tables = array_merge(['civicrm_address' => 1],
2578         $tables
2579       );
2580     }
2581
2582     // add group_contact and group table is subscription history is present
2583     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2584       $tables = array_merge([
2585         'civicrm_group' => 1,
2586         'civicrm_group_contact' => 1,
2587       ],
2588         $tables
2589       );
2590     }
2591
2592     // to handle table dependencies of components
2593     CRM_Core_Component::tableNames($tables);
2594     // to handle table dependencies of hook injected tables
2595     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2596
2597     //format the table list according to the weight
2598     $info = CRM_Core_TableHierarchy::info();
2599
2600     foreach ($tables as $key => $value) {
2601       $k = 99;
2602       if (strpos($key, '-') !== FALSE) {
2603         $keyArray = explode('-', $key);
2604         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2605       }
2606       elseif (strpos($key, '_') !== FALSE) {
2607         $keyArray = explode('_', $key);
2608         if (is_numeric(array_pop($keyArray))) {
2609           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2610         }
2611         else {
2612           $k = CRM_Utils_Array::value($key, $info, 99);
2613         }
2614       }
2615       else {
2616         $k = CRM_Utils_Array::value($key, $info, 99);
2617       }
2618       $tempTable[$k . ".$key"] = $key;
2619     }
2620     ksort($tempTable);
2621     $newTables = [];
2622     foreach ($tempTable as $key) {
2623       $newTables[$key] = $tables[$key];
2624     }
2625
2626     $tables = $newTables;
2627
2628     foreach ($tables as $name => $value) {
2629       if (!$value) {
2630         continue;
2631       }
2632
2633       if (!empty($inner[$name])) {
2634         $side = 'INNER';
2635       }
2636       elseif (!empty($right[$name])) {
2637         $side = 'RIGHT';
2638       }
2639       else {
2640         $side = 'LEFT';
2641       }
2642
2643       if ($value != 1) {
2644         // if there is already a join statement in value, use value itself
2645         if (strpos($value, 'JOIN')) {
2646           $from .= " $value ";
2647         }
2648         else {
2649           $from .= " $side JOIN $name ON ( $value ) ";
2650         }
2651         continue;
2652       }
2653
2654       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2655     }
2656     return $from;
2657   }
2658
2659   /**
2660    * Get join statements for the from clause depending on entity type
2661    *
2662    * @param string $name
2663    * @param int $mode
2664    * @param string $side
2665    * @param string $primaryLocation
2666    * @return string
2667    */
2668   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2669     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2670     switch ($name) {
2671       case 'civicrm_address':
2672         //CRM-14263 further handling of address joins further down...
2673         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2674
2675       case 'civicrm_state_province':
2676         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2677         // but that will have been implied-in by the calling function.
2678         // test cover in testContactIDQuery
2679         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2680
2681       case 'civicrm_country':
2682         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2683         // but that will have been implied-in by the calling function.
2684         // test cover in testContactIDQuery
2685         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2686
2687       case 'civicrm_phone':
2688         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2689
2690       case 'civicrm_email':
2691         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2692
2693       case 'civicrm_im':
2694         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2695
2696       case 'im_provider':
2697         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2698         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2699         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2700         return $from;
2701
2702       case 'civicrm_openid':
2703         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2704
2705       case 'civicrm_worldregion':
2706         // We can be sure from the calling function that country will already be joined in.
2707         // we really don't need world_region - we could use a pseudoconstant for it.
2708         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2709
2710       case 'civicrm_location_type':
2711         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2712
2713       case 'civicrm_group':
2714         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2715
2716       case 'civicrm_group_contact':
2717         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2718
2719       case 'civicrm_group_contact_cache':
2720         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2721
2722       case 'civicrm_activity':
2723       case 'civicrm_activity_tag':
2724       case 'activity_type':
2725       case 'activity_status':
2726       case 'parent_id':
2727       case 'civicrm_activity_contact':
2728       case 'source_contact':
2729       case 'activity_priority':
2730         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2731
2732       case 'civicrm_entity_tag':
2733         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2734         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2735
2736       case 'civicrm_note':
2737         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2738         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2739
2740       case 'civicrm_subscription_history':
2741         $from = " $side JOIN civicrm_subscription_history";
2742         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2743         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2744
2745       case 'civicrm_relationship':
2746         if (self::$_relType == 'reciprocal') {
2747           if (self::$_relationshipTempTable) {
2748             // we have a temptable to join on
2749             $tbl = self::$_relationshipTempTable;
2750             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2751           }
2752           else {
2753             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2754             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2755             return $from;
2756           }
2757         }
2758         elseif (self::$_relType == 'b') {
2759           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2760           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2761         }
2762         else {
2763           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2764           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2765         }
2766
2767       case 'civicrm_log':
2768         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2769         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2770
2771       case 'civicrm_tag':
2772         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2773
2774       case 'civicrm_grant':
2775         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2776
2777       case 'civicrm_website':
2778         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2779
2780       case 'civicrm_campaign':
2781         //Move to default case if not in either mode.
2782         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2783           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2784         }
2785         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2786           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2787         }
2788         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2789           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2790         }
2791
2792       default:
2793         $locationTypeName = '';
2794         if (strpos($name, '-address') != 0) {
2795           $locationTypeName = 'address';
2796         }
2797         elseif (strpos($name, '-phone') != 0) {
2798           $locationTypeName = 'phone';
2799         }
2800         elseif (strpos($name, '-email') != 0) {
2801           $locationTypeName = 'email';
2802         }
2803         elseif (strpos($name, '-im') != 0) {
2804           $locationTypeName = 'im';
2805         }
2806         elseif (strpos($name, '-openid') != 0) {
2807           $locationTypeName = 'openid';
2808         }
2809
2810         if ($locationTypeName) {
2811           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2812           $parts = explode('-', $name);
2813           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2814           foreach ($locationTypes as $locationTypeID => $locationType) {
2815             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2816               $locationID = $locationTypeID;
2817             }
2818           }
2819           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2820         }
2821         else {
2822           $from = CRM_Core_Component::from($name, $mode, $side);
2823         }
2824         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2825
2826         return $from;
2827     }
2828   }
2829
2830   /**
2831    * WHERE / QILL clause for deleted_contacts
2832    *
2833    * @param array $values
2834    */
2835   public function deletedContacts($values) {
2836     list($_, $_, $value, $grouping, $_) = $values;
2837     if ($value) {
2838       // *prepend* to the relevant grouping as this is quite an important factor
2839       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2840     }
2841   }
2842
2843   /**
2844    * Where / qill clause for contact_type
2845    *
2846    * @param $values
2847    *
2848    * @throws \CRM_Core_Exception
2849    */
2850   public function contactType(&$values) {
2851     list($name, $op, $value, $grouping, $wildcard) = $values;
2852
2853     $subTypes = [];
2854     $clause = [];
2855
2856     // account for search builder mapping multiple values
2857     if (!is_array($value)) {
2858       $values = self::parseSearchBuilderString($value, 'String');
2859       if (is_array($values)) {
2860         $value = array_flip($values);
2861       }
2862     }
2863
2864     if (is_array($value)) {
2865       foreach ($value as $k => $v) {
2866         // fix for CRM-771
2867         if ($k) {
2868           $subType = NULL;
2869           $contactType = $k;
2870           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2871             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2872           }
2873
2874           if (!empty($subType)) {
2875             $subTypes[$subType] = 1;
2876           }
2877           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2878         }
2879       }
2880     }
2881     else {
2882       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2883       $contactType = $contactTypeANDSubType[0];
2884       $subType = $contactTypeANDSubType[1] ?? NULL;
2885       if (!empty($subType)) {
2886         $subTypes[$subType] = 1;
2887       }
2888       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2889     }
2890
2891     // fix for CRM-771
2892     if (!empty($clause)) {
2893       $quill = $clause;
2894       if ($op == 'IN' || $op == 'NOT IN') {
2895         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2896       }
2897       else {
2898         $type = array_pop($clause);
2899         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2900       }
2901
2902       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2903
2904       if (!empty($subTypes)) {
2905         $this->includeContactSubTypes($subTypes, $grouping);
2906       }
2907     }
2908   }
2909
2910   /**
2911    * Where / qill clause for contact_sub_type.
2912    *
2913    * @param array $values
2914    */
2915   public function contactSubType(&$values) {
2916     list($name, $op, $value, $grouping, $wildcard) = $values;
2917     $this->includeContactSubTypes($value, $grouping, $op);
2918   }
2919
2920   /**
2921    * @param $value
2922    * @param $grouping
2923    * @param string $op
2924    *
2925    * @throws \CRM_Core_Exception
2926    */
2927   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2928
2929     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2930       $op = key($value);
2931       $value = $value[$op];
2932     }
2933
2934     $clause = [];
2935     $alias = "contact_a.contact_sub_type";
2936     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2937
2938     $op = str_replace('IN', 'LIKE', $op);
2939     $op = str_replace('=', 'LIKE', $op);
2940     $op = str_replace('!', 'NOT ', $op);
2941
2942     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2943       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2944     }
2945     elseif (is_array($value)) {
2946       foreach ($value as $k => $v) {
2947         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2948       }
2949     }
2950     else {
2951       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2952     }
2953
2954     if (!empty($clause)) {
2955       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2956     }
2957     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2958   }
2959
2960   /**
2961    * Where / qill clause for groups.
2962    *
2963    * @param $values
2964    *
2965    * @throws \CRM_Core_Exception
2966    * @throws \Exception
2967    */
2968   public function group($values) {
2969     list($name, $op, $value, $grouping, $wildcard) = $values;
2970
2971     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2972     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2973       $op = key($value);
2974       $value = $value[$op];
2975     }
2976     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2977     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2978     // even one group might equate to multiple when looking at children so IN is simpler.
2979     // @todo - also look at != casting but there are rows below to review.
2980     $opReplacements = [
2981       'IS EMPTY' => 'IS NULL',
2982       'IS NOT EMPTY' => 'IS NOT NULL',
2983       '=' => 'IN',
2984     ];
2985     if (isset($opReplacements[$op])) {
2986       $op = $opReplacements[$op];
2987     }
2988
2989     if (strpos($op, 'NULL')) {
2990       $value = NULL;
2991     }
2992
2993     if (is_array($value) && count($value) > 1) {
2994       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2995         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2996       }
2997       $this->_useDistinct = TRUE;
2998     }
2999
3000     if (isset($value)) {
3001       $value = CRM_Utils_Array::value($op, $value, $value);
3002     }
3003
3004     if ($name === 'group_type') {
3005       $value = array_keys($this->getGroupsFromTypeCriteria($value));
3006     }
3007
3008     $regularGroupIDs = $smartGroupIDs = [];
3009     foreach ((array) $value as $id) {
3010       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3011         $smartGroupIDs[] = (int) $id;
3012       }
3013       else {
3014         $regularGroupIDs[] = (int) trim($id);
3015       }
3016     }
3017     $hasNonSmartGroups = count($regularGroupIDs);
3018
3019     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3020
3021     $statusJoinClause = $this->getGroupStatusClause($grouping);
3022     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3023     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3024     $groupClause = [];
3025     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3026       // include child groups IDs if any
3027       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3028       foreach ($childGroupIds as $key => $id) {
3029         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3030           $smartGroupIDs[] = $id;
3031           unset($childGroupIds[$key]);
3032         }
3033       }
3034       if (count($childGroupIds)) {
3035         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3036       }
3037
3038       if (empty($regularGroupIDs)) {
3039         if ($isGroupStatusSearch) {
3040           $regularGroupIDs = $smartGroupIDs;
3041         }
3042         // If it is still empty we want a filter that blocks all results.
3043         if (empty($regularGroupIDs)) {
3044           $regularGroupIDs = [0];
3045         }
3046       }
3047
3048       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3049       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3050
3051       // @todo consider just casting != to NOT IN & handling both together.
3052       if ($op === '!=') {
3053         $groupIds = '';
3054         if (!empty($regularGroupIDs)) {
3055           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3056         }
3057         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3058       }
3059       else {
3060         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3061       }
3062       $groupClause[] = "( {$clause} )";
3063
3064       if ($statusJoinClause) {
3065         $joinClause[] = "{$gcTable}.$statusJoinClause";
3066       }
3067       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3068     }
3069
3070     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3071     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3072     // as that info is in the group_contact table.
3073     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3074       $this->_groupUniqueKey = uniqid();
3075       $this->_groupKeys[] = $this->_groupUniqueKey;
3076       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3077       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3078       if (!empty($groupContactCacheClause)) {
3079         if ($isNotOp) {
3080           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3081           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3082           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3083           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3084           if (strpos($op, 'IN') !== FALSE) {
3085             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3086           }
3087           else {
3088             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3089           }
3090         }
3091         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3092       }
3093     }
3094
3095     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3096     if (!empty($groupClause)) {
3097       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3098     }
3099
3100     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3101     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3102     if (strpos($op, 'NULL') === FALSE) {
3103       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3104     }
3105   }
3106
3107   /**
3108    * @return array
3109    */
3110   public function getGroupCacheTableKeys() {
3111     return $this->_groupKeys;
3112   }
3113
3114   /**
3115    * Function translates selection of group type into a list of groups.
3116    * @param $value
3117    *
3118    * @return array
3119    */
3120   public function getGroupsFromTypeCriteria($value) {
3121     $groupIds = [];
3122     foreach ((array) $value as $groupTypeValue) {
3123       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3124       $groupIds = ($groupIds + $groupList);
3125     }
3126     return $groupIds;
3127   }
3128
3129   /**
3130    * Prime smart group cache for smart groups in the search, and join
3131    * civicrm_group_contact_cache table into the query.
3132    *
3133    * @param array $groups IDs of groups specified in search criteria.
3134    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3135    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3136    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3137    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3138    *
3139    * @return string WHERE clause component for smart group criteria.
3140    * @throws \CRM_Core_Exception
3141    */
3142   public function addGroupContactCache($groups, $tableAlias, $joinTable, $op, $joinColumn = 'id') {
3143     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3144     $groupsIds = $groups;
3145
3146     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3147     if (!empty($operator[$op]) && is_array($groups)) {
3148       $op = $operator[$op];
3149     }
3150     if (!$isNullOp && !$groups) {
3151       return NULL;
3152     }
3153     elseif (strpos($op, 'IN') !== FALSE) {
3154       $groups = [$op => $groups];
3155     }
3156     elseif (is_array($groups) && count($groups)) {
3157       $groups = ['IN' => $groups];
3158     }
3159
3160     // Find all the groups that are part of a saved search.
3161     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3162     $sql = "
3163 SELECT id, cache_date, saved_search_id, children
3164 FROM   civicrm_group
3165 WHERE  $smartGroupClause
3166   AND  ( saved_search_id != 0
3167    OR    saved_search_id IS NOT NULL
3168    OR    children IS NOT NULL )
3169 ";
3170
3171     $group = CRM_Core_DAO::executeQuery($sql);
3172
3173     while ($group->fetch()) {
3174       $this->_useDistinct = TRUE;
3175       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3176         CRM_Contact_BAO_GroupContactCache::load($group);
3177       }
3178     }
3179     if ($group->N == 0 && $op != 'NOT IN') {
3180       return NULL;
3181     }
3182
3183     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3184
3185     if ($op == 'NOT IN') {
3186       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3187     }
3188     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3189   }
3190
3191   /**
3192    * Where / qill clause for cms users
3193    *
3194    * @param $values
3195    */
3196   public function ufUser(&$values) {
3197     list($name, $op, $value, $grouping, $wildcard) = $values;
3198
3199     if ($value == 1) {
3200       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3201
3202       $this->_qill[$grouping][] = ts('CMS User');
3203     }
3204     elseif ($value == 0) {
3205       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3206
3207       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3208       $this->_qill[$grouping][] = ts('Not a CMS User');
3209     }
3210   }
3211
3212   /**
3213    * All tag search specific.
3214    *
3215    * @param array $values
3216    *
3217    * @throws \CRM_Core_Exception
3218    */
3219   public function tagSearch(&$values) {
3220     list($name, $op, $value, $grouping, $wildcard) = $values;
3221
3222     $op = "LIKE";
3223     $value = "%{$value}%";
3224     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3225
3226     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3227     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3228
3229     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3230     $tTable = "`civicrm_tag-" . uniqid() . "`";
3231
3232     if ($useAllTagTypes[2]) {
3233       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3234         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3235             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3236
3237       // search tag in cases
3238       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3239       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3240       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3241         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3242             LEFT JOIN civicrm_case
3243             ON (civicrm_case_contact.case_id = civicrm_case.id
3244                 AND civicrm_case.is_deleted = 0 )
3245             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3246             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3247       // search tag in activities
3248       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3249       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3250       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3251       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3252
3253       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3254         = " LEFT JOIN civicrm_activity_contact
3255             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3256             LEFT JOIN civicrm_activity
3257             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3258             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3259             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3260             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3261
3262       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3263       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3264     }
3265     else {
3266       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3267       $tTable = "`civicrm_tag-" . uniqid() . "`";
3268       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3269       {$etTable}.entity_table = 'civicrm_contact' )
3270                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3271
3272       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3273       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3274     }
3275   }
3276
3277   /**
3278    * Where / qill clause for tag.
3279    *
3280    * @param array $values
3281    *
3282    * @throws \CRM_Core_Exception
3283    */
3284   public function tag(&$values) {
3285     list($name, $op, $value, $grouping, $wildcard) = $values;
3286
3287     // API/Search Builder format array(operator => array(values))
3288     if (is_array($value)) {
3289       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3290         $op = key($value);
3291         $value = $value[$op];
3292       }
3293       if (count($value) > 1) {
3294         $this->_useDistinct = TRUE;
3295       }
3296     }
3297
3298     if (strpos($op, 'NULL') || strpos($op, 'EMPTY')) {
3299       $value = NULL;
3300     }
3301
3302     $tagTree = CRM_Core_BAO_Tag::getChildTags();
3303     foreach ((array) $value as $tagID) {
3304       if (!empty($tagTree[$tagID])) {
3305         // make sure value is an array here (see CORE-2502)
3306         $value = array_unique(array_merge((array) $value, $tagTree[$tagID]));
3307       }
3308     }
3309
3310     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3311
3312     // implode array, then remove all spaces
3313     $value = str_replace(' ', '', implode(',', (array) $value));
3314     if (!empty($value)) {
3315       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3316     }
3317
3318     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3319     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3320
3321     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3322
3323     if (!empty($useAllTagTypes[2])) {
3324       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3325         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3326
3327       // search tag in cases
3328       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3329       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3330       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3331
3332       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3333         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3334             LEFT JOIN civicrm_case
3335             ON (civicrm_case_contact.case_id = civicrm_case.id
3336                 AND civicrm_case.is_deleted = 0 )
3337             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3338       // search tag in activities
3339       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3340       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3341         = " LEFT JOIN civicrm_activity_contact
3342             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3343             LEFT JOIN civicrm_activity
3344             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3345             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3346             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3347
3348       // CRM-10338
3349       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3350         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3351       }
3352       else {
3353         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3354       }
3355     }
3356     else {
3357       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3358         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3359
3360       // CRM-10338
3361       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3362         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3363         $op = str_replace('EMPTY', 'NULL', $op);
3364         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3365       }
3366       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3367       elseif ($op == '!=') {
3368         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3369       }
3370       elseif ($op == '=' || strstr($op, 'IN')) {
3371         $op = ($op == '=') ? 'IN' : $op;
3372         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3373       }
3374     }
3375     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3376   }
3377
3378   /**
3379    * Where/qill clause for notes
3380    *
3381    * @param array $values
3382    *
3383    * @throws \CRM_Core_Exception
3384    */
3385   public function notes(&$values) {
3386     list($name, $op, $value, $grouping, $wildcard) = $values;
3387
3388     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3389     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3390     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3391
3392     $this->_useDistinct = TRUE;
3393
3394     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3395       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3396
3397     $n = trim($value);
3398     $value = CRM_Core_DAO::escapeString($n);
3399     if ($wildcard) {
3400       if (strpos($value, '%') === FALSE) {
3401         $value = "%$value%";
3402       }
3403       $op = 'LIKE';
3404     }
3405     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3406       $value = NULL;
3407     }
3408
3409     $label = NULL;
3410     $clauses = [];
3411     if ($noteOption % 2 == 0) {
3412       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3413       $label = ts('Note: Body Only');
3414     }
3415     if ($noteOption % 3 == 0) {
3416       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3417       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3418     }
3419     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3420     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3421     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3422   }
3423
3424   /**
3425    * @param string $name
3426    * @param $op
3427    * @param $grouping
3428    *
3429    * @return bool
3430    */
3431   public function nameNullOrEmptyOp($name, $op, $grouping) {
3432     switch ($op) {
3433       case 'IS NULL':
3434       case 'IS NOT NULL':
3435         $this->_where[$grouping][] = "contact_a.$name $op";
3436         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3437         return TRUE;
3438
3439       case 'IS EMPTY':
3440         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3441         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3442         return TRUE;
3443
3444       case 'IS NOT EMPTY':
3445         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3446         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3447         return TRUE;
3448
3449       default:
3450         return FALSE;
3451     }
3452   }
3453
3454   /**
3455    * Where / qill clause for sort_name
3456    *
3457    * @param array $values
3458    */
3459   public function sortName(&$values) {
3460     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3461
3462     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3463     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3464       return;
3465     }
3466
3467     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3468
3469     if (!strlen($value)) {
3470       return;
3471     }
3472
3473     $config = CRM_Core_Config::singleton();
3474
3475     $sub = [];
3476
3477     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3478     $subGlue = ' OR ';
3479
3480     $firstChar = substr($value, 0, 1);
3481     $lastChar = substr($value, -1, 1);
3482     $quotes = ["'", '"'];
3483     // If string is quoted, strip quotes and otherwise don't alter it
3484     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3485       $value = trim($value, implode('', $quotes));
3486     }
3487     // Replace spaces with wildcards for a LIKE operation
3488     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3489     // Also need to check if there is space in between sort name.
3490     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') !== FALSE) {
3491       $value = str_replace(' ', '%', $value);
3492     }
3493     $value = CRM_Core_DAO::escapeString(trim($value));
3494     if (strlen($value)) {
3495       $fieldsub = [];
3496       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3497       if ($fieldName == 'sort_name') {
3498         $wc = "contact_a.sort_name";
3499       }
3500       else {
3501         $wc = "contact_a.display_name";
3502       }
3503       $fieldsub[] = " ( $wc $op $value )";
3504       if ($config->includeNickNameInName) {
3505         $wc = "contact_a.nick_name";
3506         $fieldsub[] = " ( $wc $op $value )";
3507       }
3508       if ($config->includeEmailInName) {
3509         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3510       }
3511       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3512     }
3513
3514     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3515
3516     $this->_where[$grouping][] = $sub;
3517     if ($config->includeEmailInName) {
3518       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3519       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3520     }
3521     else {
3522       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3523     }
3524   }
3525
3526   /**
3527    * Where/qill clause for greeting fields.
3528    *
3529    * @param array $values
3530    *
3531    * @throws \CRM_Core_Exception
3532    */
3533   public function greetings(&$values) {
3534     list($name, $op, $value, $grouping, $wildcard) = $values;
3535     $name .= '_display';
3536
3537     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3538     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3539     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3540   }
3541
3542   /**
3543    * Where / qill clause for email
3544    *
3545    * @param array $values
3546    * @param string $isForcePrimaryOnly
3547    *
3548    * @throws \CRM_Core_Exception
3549    */
3550   protected function email(&$values, $isForcePrimaryOnly) {
3551     list($name, $op, $value, $grouping, $wildcard) = $values;
3552     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3553
3554     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3555     // so following code is use build whereClause for contact's primart email id
3556     if (!empty($isForcePrimaryOnly)) {
3557       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3558     }
3559     // @todo - this should come from the $this->_fields array
3560     $dbName = $name === 'email_id' ? 'id' : $name;
3561
3562     if (is_array($value) || $name === 'email_id') {
3563       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3564       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3565       return;
3566     }
3567
3568     // Is this ever hit now? Ideally ensure always an array & handle above.
3569     $n = trim($value);
3570     if ($n) {
3571       if (substr($n, 0, 1) == '"' &&
3572         substr($n, -1, 1) == '"'
3573       ) {
3574         $n = substr($n, 1, -1);
3575         $value = CRM_Core_DAO::escapeString($n);
3576         $op = '=';
3577       }
3578       else {
3579         $value = self::getWildCardedValue($wildcard, $op, $n);
3580       }
3581       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3582       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3583     }
3584     else {
3585       $this->_qill[$grouping][] = ts('Email') . " $op ";
3586       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3587     }
3588   }
3589
3590   /**
3591    * Where / qill clause for phone number
3592    *
3593    * @param array $values
3594    *
3595    * @throws \CRM_Core_Exception
3596    */
3597   public function phone_numeric(&$values) {
3598     list($name, $op, $value, $grouping, $wildcard) = $values;
3599     // Strip non-numeric characters; allow wildcards
3600     $number = preg_replace('/[^\d%]/', '', $value);
3601     if ($number) {
3602       if (strpos($number, '%') === FALSE) {
3603         $number = "%$number%";
3604       }
3605
3606       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3607       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3608       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3609     }
3610   }
3611
3612   /**
3613    * Where / qill clause for phone type/location
3614    *
3615    * @param array $values
3616    *
3617    * @throws \CRM_Core_Exception
3618    */
3619   public function phone_option_group($values) {
3620     list($name, $op, $value, $grouping, $wildcard) = $values;
3621     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3622     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3623     $optionName = $options[$value];
3624     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3625     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3626     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3627   }
3628
3629   /**
3630    * Where / qill clause for street_address.
3631    *
3632    * @param array $values
3633    *
3634    * @throws \CRM_Core_Exception
3635    */
3636   public function street_address(&$values) {
3637     list($name, $op, $value, $grouping) = $values;
3638
3639     if (!$op) {
3640       $op = 'LIKE';
3641     }
3642
3643     $n = trim($value);
3644
3645     if ($n) {
3646       if (strpos($value, '%') === FALSE) {
3647         // only add wild card if not there
3648         $value = "%{$value}%";
3649       }
3650       $op = 'LIKE';
3651       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3652       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3653     }
3654     else {
3655       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3656       $this->_qill[$grouping][] = ts('Street') . " $op ";
3657     }
3658
3659     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3660   }
3661
3662   /**
3663    * Where / qill clause for street_unit.
3664    *
3665    * @param array $values
3666    *
3667    * @throws \CRM_Core_Exception
3668    */
3669   public function street_number(&$values) {
3670     list($name, $op, $value, $grouping, $wildcard) = $values;
3671
3672     if (!$op) {
3673       $op = '=';
3674     }
3675
3676     $n = trim($value);
3677
3678     if (strtolower($n) == 'odd') {
3679       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3680       $this->_qill[$grouping][] = ts('Street Number is odd');
3681     }
3682     elseif (strtolower($n) == 'even') {
3683       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3684       $this->_qill[$grouping][] = ts('Street Number is even');
3685     }
3686     else {
3687       $value = $n;
3688       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3689       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3690     }
3691
3692     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3693   }
3694
3695   /**
3696    * Where / qill clause for sorting by character.
3697    *
3698    * @param array $values
3699    */
3700   public function sortByCharacter(&$values) {
3701     list($name, $op, $value, $grouping, $wildcard) = $values;
3702
3703     $name = trim($value);
3704     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3705     $this->_where[$grouping][] = $cond;
3706     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3707   }
3708
3709   /**
3710    * Where / qill clause for including contact ids.
3711    */
3712   public function includeContactIDs() {
3713     if (!$this->_includeContactIds || empty($this->_params)) {
3714       return;
3715     }
3716
3717     $contactIds = [];
3718     foreach ($this->_params as $id => $values) {
3719       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3720         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3721       }
3722     }
3723     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3724     if (!empty($contactIds)) {
3725       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3726     }
3727   }
3728
3729   /**
3730    * Where / qill clause for postal code.
3731    *
3732    * @param array $values
3733    *
3734    * @throws \CRM_Core_Exception
3735    */
3736   public function postalCode(&$values) {
3737     // skip if the fields dont have anything to do with postal_code
3738     if (empty($this->_fields['postal_code'])) {
3739       return;
3740     }
3741
3742     list($name, $op, $value, $grouping, $wildcard) = $values;
3743
3744     // Handle numeric postal code range searches properly by casting the column as numeric
3745     if (is_numeric($value)) {
3746       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3747       $val = CRM_Utils_Type::escape($value, 'Integer');
3748     }
3749     else {
3750       $field = 'civicrm_address.postal_code';
3751       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3752       if (!is_array($value)) {
3753         $val = CRM_Utils_Type::escape($value, 'String');
3754       }
3755       else {
3756         // Do we need to escape values here? I would expect buildClause does.
3757         $val = $value;
3758       }
3759     }
3760
3761     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3762
3763     if ($name == 'postal_code') {
3764       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3765       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3766     }
3767     elseif ($name == 'postal_code_low') {
3768       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3769       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3770     }
3771     elseif ($name == 'postal_code_high') {
3772       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3773       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3774     }
3775   }
3776
3777   /**
3778    * Where / qill clause for location type.
3779    *
3780    * @param array $values
3781    * @param null $status
3782    *
3783    * @return string
3784    */
3785   public function locationType(&$values, $status = NULL) {
3786     list($name, $op, $value, $grouping, $wildcard) = $values;
3787
3788     if (is_array($value)) {
3789       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3790       $this->_tables['civicrm_address'] = 1;
3791       $this->_whereTables['civicrm_address'] = 1;
3792
3793       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3794       $names = [];
3795       foreach ($value as $id) {
3796         $names[] = $locationType[$id];
3797       }
3798
3799       $this->_primaryLocation = FALSE;
3800
3801       if (!$status) {
3802         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3803       }
3804       else {
3805         return implode(' ' . ts('or') . ' ', $names);
3806       }
3807     }
3808   }
3809
3810   /**
3811    * @param $values
3812    * @param bool $fromStateProvince
3813    *
3814    * @return array|NULL
3815    * @throws \CRM_Core_Exception
3816    */
3817   public function country(&$values, $fromStateProvince = TRUE) {
3818     list($name, $op, $value, $grouping, $wildcard) = $values;
3819
3820     if (!$fromStateProvince) {
3821       $stateValues = $this->getWhereValues('state_province', $grouping);
3822       if (!empty($stateValues)) {
3823         // return back to caller if there are state province values
3824         // since that handles this case
3825         return NULL;
3826       }
3827     }
3828
3829     $countryClause = $countryQill = NULL;
3830     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3831       $this->_tables['civicrm_address'] = 1;
3832       $this->_whereTables['civicrm_address'] = 1;
3833
3834       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3835       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3836       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3837
3838       if (!$fromStateProvince) {
3839         $this->_where[$grouping][] = $countryClause;
3840         $this->_qill[$grouping][] = $countryQill;
3841       }
3842     }
3843
3844     if ($fromStateProvince) {
3845       if (!empty($countryClause)) {
3846         return [
3847           $countryClause,
3848           " ...AND... " . $countryQill,
3849         ];
3850       }
3851       else {
3852         return [NULL, NULL];
3853       }
3854     }
3855   }
3856
3857   /**
3858    * Where / qill clause for county (if present).
3859    *
3860    * @param array $values
3861    * @param null $status
3862    *
3863    * @return string
3864    */
3865   public function county(&$values, $status = NULL) {
3866     list($name, $op, $value, $grouping, $wildcard) = $values;
3867
3868     if (!is_array($value)) {
3869       // force the county to be an array
3870       $value = [$value];
3871     }
3872
3873     // check if the values are ids OR names of the counties
3874     $inputFormat = 'id';
3875     foreach ($value as $v) {
3876       if (!is_numeric($v)) {
3877         $inputFormat = 'name';
3878         break;
3879       }
3880     }
3881     $names = [];
3882     if ($op == '=') {
3883       $op = 'IN';
3884     }
3885     elseif ($op == '!=') {
3886       $op = 'NOT IN';
3887     }
3888     else {
3889       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3890       $op = str_replace('EMPTY', 'NULL', $op);
3891     }
3892
3893     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3894       $clause = "civicrm_address.county_id $op";
3895     }
3896     elseif ($inputFormat == 'id') {
3897       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3898
3899       $county = CRM_Core_PseudoConstant::county();
3900       foreach ($value as $id) {
3901         $names[] = $county[$id] ?? NULL;
3902       }
3903     }
3904     else {
3905       $inputClause = [];
3906       $county = CRM_Core_PseudoConstant::county();
3907       foreach ($value as $name) {
3908         $name = trim($name);
3909         $inputClause[] = CRM_Utils_Array::key($name, $county);
3910       }
3911       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3912       $names = $value;
3913     }
3914     $this->_tables['civicrm_address'] = 1;
3915     $this->_whereTables['civicrm_address'] = 1;
3916
3917     $this->_where[$grouping][] = $clause;
3918     if (!$status) {
3919       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3920     }
3921     else {
3922       return implode(' ' . ts('or') . ' ', $names);
3923     }
3924   }
3925
3926   /**
3927    * Where / qill clause for state/province AND country (if present).
3928    *
3929    * @param array $values
3930    * @param null $status
3931    *
3932    * @return string
3933    * @throws \CRM_Core_Exception
3934    */
3935   public function stateProvince(&$values, $status = NULL) {
3936     list($name, $op, $value, $grouping, $wildcard) = $values;
3937
3938     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3939     $this->_tables['civicrm_address'] = 1;
3940     $this->_whereTables['civicrm_address'] = 1;
3941
3942     $countryValues = $this->getWhereValues('country', $grouping);
3943     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3944     if ($countryClause) {
3945       $clause = "( $stateClause AND $countryClause )";
3946     }
3947     else {
3948       $clause = $stateClause;
3949     }
3950
3951     $this->_where[$grouping][] = $clause;
3952     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3953     if (!$status) {
3954       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3955     }
3956     else {
3957       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3958     }
3959   }
3960
3961   /**
3962    * Where / qill clause for change log.
3963    *
3964    * @param array $values
3965    */
3966   public function changeLog(&$values) {
3967     list($name, $op, $value, $grouping, $wildcard) = $values;
3968
3969     $targetName = $this->getWhereValues('changed_by', $grouping);
3970     if (!$targetName) {
3971       return;
3972     }
3973
3974     $name = trim($targetName[2]);
3975     $name = CRM_Core_DAO::escapeString($name);
3976     $name = $targetName[4] ? "%$name%" : $name;
3977     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3978     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3979     $fieldTitle = ts('Altered By');
3980
3981     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3982     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3983       1 => $fieldTitle,
3984       2 => $qillop,
3985       3 => $qillVal,
3986     ]);
3987   }
3988
3989   /**
3990    * @param $values
3991    *
3992    * @throws \CRM_Core_Exception
3993    */
3994   public function modifiedDates($values) {
3995     $this->_useDistinct = TRUE;
3996     CRM_Core_Error::deprecatedWarning('function should not be reachable');
3997     // CRM-11281, default to added date if not set
3998     $fieldTitle = ts('Added Date');
3999     $fieldName = 'created_date';
4000     foreach (array_keys($this->_params) as $id) {
4001       if ($this->_params[$id][0] == 'log_date') {
4002         if ($this->_params[$id][2] == 2) {
4003           $fieldTitle = ts('Modified Date');
4004           $fieldName = 'modified_date';
4005         }
4006       }
4007     }
4008
4009     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
4010
4011     self::$_openedPanes[ts('Change Log')] = TRUE;
4012   }
4013
4014   /**
4015    * @param $values
4016    *
4017    * @throws \CRM_Core_Exception
4018    */
4019   public function demographics(&$values) {
4020     list($name, $op, $value, $grouping, $wildcard) = $values;
4021
4022     if (($name == 'age_low') || ($name == 'age_high')) {
4023       $this->ageRangeQueryBuilder($values,
4024         'contact_a', 'age', 'birth_date', ts('Age')
4025       );
4026     }
4027     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4028
4029       $this->dateQueryBuilder($values,
4030         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4031       );
4032     }
4033     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4034
4035       $this->dateQueryBuilder($values,
4036         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4037       );
4038     }
4039
4040     self::$_openedPanes[ts('Demographics')] = TRUE;
4041   }
4042
4043   /**
4044    * @param $values
4045    */
4046   public function privacy(&$values) {
4047     list($name, $op, $value, $grouping) = $values;
4048     if (is_array($value)) {
4049       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4050         $op = key($value);
4051         $value = $value[$op];
4052       }
4053     }
4054     $field = $this->_fields[$name] ?? NULL;
4055     CRM_Utils_Type::validate($value, 'Integer');
4056     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4057     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4058     $title = $field ? $field['title'] : $name;
4059     $this->_qill[$grouping][] = "$title $op $value";
4060   }
4061
4062   /**
4063    * @param $values
4064    */
4065   public function privacyOptions($values) {
4066     list($name, $op, $value, $grouping, $wildcard) = $values;
4067
4068     if (empty($value) || !is_array($value)) {
4069       return;
4070     }
4071
4072     // get the operator and toggle values
4073     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4074     $operator = 'OR';
4075     if ($opValues &&
4076       strtolower($opValues[2] == 'AND')
4077     ) {
4078       // @todo this line is logially unreachable
4079       $operator = 'AND';
4080     }
4081
4082     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4083     $compareOP = '!';
4084     if ($toggleValues &&
4085       $toggleValues[2] == 2
4086     ) {
4087       $compareOP = '';
4088     }
4089
4090     $clauses = [];
4091     $qill = [];
4092     foreach ($value as $dontCare => $pOption) {
4093       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4094       $field = $this->_fields[$pOption] ?? NULL;
4095       $title = $field ? $field['title'] : $pOption;
4096       $qill[] = " $title = 1 ";
4097     }
4098
4099     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4100     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4101   }
4102
4103   /**
4104    * @param $values
4105    *
4106    * @throws \CRM_Core_Exception
4107    */
4108   public function preferredCommunication(&$values) {
4109     list($name, $op, $value, $grouping, $wildcard) = $values;
4110
4111     if (!is_array($value)) {
4112       $value = str_replace(['(', ')'], '', explode(",", $value));
4113     }
4114     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4115       $op = key($value);
4116       $value = $value[$op];
4117     }
4118     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4119
4120     if (self::caseImportant($op)) {
4121       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4122       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4123       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4124     }
4125
4126     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4127     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4128   }
4129
4130   /**
4131    * Where / qill clause for relationship.
4132    *
4133    * @param array $values
4134    */
4135   public function relationship(&$values) {
4136     list($name, $op, $value, $grouping, $wildcard) = $values;
4137     if ($this->_relationshipValuesAdded) {
4138       return;
4139     }
4140     // also get values array for relation_target_name
4141     // for relationship search we always do wildcard
4142     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4143     $description = $this->getWhereValues('relation_description', $grouping);
4144     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4145     $relStatus = $this->getWhereValues('relation_status', $grouping);
4146     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4147
4148     $nameClause = $name = NULL;
4149     if ($targetName) {
4150       $name = trim($targetName[2]);
4151       if (substr($name, 0, 1) == '"' &&
4152         substr($name, -1, 1) == '"'
4153       ) {
4154         $name = substr($name, 1, -1);
4155         $name = CRM_Core_DAO::escapeString($name);
4156         $nameClause = "= '$name'";
4157       }
4158       else {
4159         $name = CRM_Core_DAO::escapeString($name);
4160         $nameClause = "LIKE '%{$name}%'";
4161       }
4162     }
4163
4164     $relTypes = $relTypesIds = [];
4165     if (!empty($relationType)) {
4166       $relationType[2] = (array) $relationType[2];
4167       foreach ($relationType[2] as $relType) {
4168         $rel = explode('_', $relType);
4169         self::$_relType = $rel[1];
4170         $params = ['id' => $rel[0]];
4171         $typeValues = [];
4172         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4173         if (!empty($rTypeValue)) {
4174           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4175             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4176             self::$_relType = 'reciprocal';
4177           }
4178           $relTypesIds[] = $rel[0];
4179           $relTypes[] = $relType;
4180         }
4181       }
4182     }
4183
4184     // if we are creating a temp table we build our own where for the relationship table
4185     $relationshipTempTable = NULL;
4186     if (self::$_relType == 'reciprocal') {
4187       $where = [];
4188       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4189         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4190         ->getName();
4191       if ($nameClause) {
4192         $where[$grouping][] = " sort_name $nameClause ";
4193       }
4194       $groupJoinTable = "civicrm_relationship";
4195       $groupJoinColumn = "contact_id_alt";
4196     }
4197     else {
4198       $where = &$this->_where;
4199       if ($nameClause) {
4200         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4201       }
4202       $groupJoinTable = "contact_b";
4203       $groupJoinColumn = "id";
4204     }
4205     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4206     if ($nameClause || !$targetGroup) {
4207       if (!empty($relationType)) {
4208         $relQill = '';
4209         foreach ($relTypes as $rel) {
4210           if (!empty($relQill)) {
4211             $relQill .= ' OR ';
4212           }
4213           $relQill .= $allRelationshipType[$rel];
4214         }
4215         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4216       }
4217       elseif ($name) {
4218         $this->_qill[$grouping][] = $name;
4219       }
4220     }
4221
4222     //check to see if the target contact is in specified group
4223     if ($targetGroup) {
4224       //add contacts from static groups
4225       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4226         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4227       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4228         implode(",", $targetGroup[2]) . ") ) ";
4229
4230       //add contacts from saved searches
4231       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4232
4233       //set the group where clause
4234       if ($ssWhere) {
4235         $groupWhere[] = "( " . $ssWhere . " )";
4236       }
4237       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4238
4239       //Get the names of the target groups for the qill
4240       $groupNames = CRM_Core_PseudoConstant::group();
4241       $qillNames = [];
4242       foreach ($targetGroup[2] as $groupId) {
4243         if (array_key_exists($groupId, $groupNames)) {
4244           $qillNames[] = $groupNames[$groupId];
4245         }
4246       }
4247       if (!empty($relationType)) {
4248         $relQill = '';
4249         foreach ($relTypes as $rel) {
4250           if (!empty($relQill)) {
4251             $relQill .= ' OR ';
4252           }
4253           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4254         }
4255         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4256       }
4257       else {
4258         $this->_qill[$grouping][] = implode(", ", $qillNames);
4259       }
4260     }
4261
4262     // Description
4263     if (!empty($description[2]) && trim($description[2])) {
4264       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4265       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4266       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4267     }
4268
4269     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4270     $today = date('Ymd');
4271     //check for active, inactive and all relation status
4272     if (empty($relStatus[2])) {
4273       $where[$grouping][] = "(
4274 civicrm_relationship.is_active = 1 AND
4275 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4276 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4277 )";
4278       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4279     }
4280     elseif ($relStatus[2] == 1) {
4281       $where[$grouping][] = "(
4282 civicrm_relationship.is_active = 0 OR
4283 civicrm_relationship.end_date < {$today} OR
4284 civicrm_relationship.start_date > {$today}
4285 )";
4286       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4287     }
4288
4289     $onlyDeleted = 0;
4290     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4291       $onlyDeleted = 1;
4292     }
4293     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4294
4295     $this->addRelationshipPermissionClauses($grouping, $where);
4296     $this->addRelationshipDateClauses($grouping, $where);
4297     $this->addRelationshipActivePeriodClauses($grouping, $where);
4298     if (!empty($relTypes)) {
4299       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4300     }
4301     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4302     $this->_useDistinct = TRUE;
4303     $this->_relationshipValuesAdded = TRUE;
4304     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4305     // join on that,
4306     if ($relationshipTempTable) {
4307       $whereClause = '';
4308       if (!empty($where[$grouping])) {
4309         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4310         $whereClause = str_replace('contact_b', 'c', $whereClause);
4311       }
4312       $sql = "
4313         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4314           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4315             FROM civicrm_relationship
4316             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4317             $whereClause )
4318           UNION
4319             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4320             FROM civicrm_relationship
4321             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4322             $whereClause )
4323       ";
4324       CRM_Core_DAO::executeQuery($sql);
4325     }
4326   }
4327
4328   /**
4329    * Add relationship permission criteria to where clause.
4330    *
4331    * @param string $grouping
4332    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4333    *   Not the main query.
4334    */
4335   public function addRelationshipPermissionClauses($grouping, &$where) {
4336     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4337     if ($relPermission) {
4338       if (!is_array($relPermission[2])) {
4339         // this form value was scalar in previous versions of Civi
4340         $relPermission[2] = [$relPermission[2]];
4341       }
4342       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4343
4344       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4345
4346       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4347       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4348     }
4349   }
4350
4351   /**
4352    * Add start & end date criteria in
4353    * @param string $grouping
4354    * @param array $where
4355    *   = array to add where clauses to, in case you are generating a temp table.
4356    * not the main query.
4357    */
4358   public function addRelationshipDateClauses($grouping, &$where) {
4359     foreach (['start_date', 'end_date'] as $dateField) {
4360       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4361       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4362       if (!empty($dateValueLow)) {
4363         $date = date('Ymd', strtotime($dateValueLow[2]));
4364         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4365         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4366       }
4367       if (!empty($dateValueHigh)) {
4368         $date = date('Ymd', strtotime($dateValueHigh[2]));
4369         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4370         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4371       }
4372     }
4373   }
4374
4375   /**
4376    * Add start & end active period criteria in
4377    * @param string $grouping
4378    * @param array $where
4379    *   = array to add where clauses to, in case you are generating a temp table.
4380    * not the main query.
4381    */
4382   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4383     $dateValues = [];
4384     $dateField = 'active_period_date';
4385
4386     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4387     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4388     $dateValueLowFormated = $dateValueHighFormated = NULL;
4389     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4390       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4391       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4392       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4393     }
4394     elseif (!empty($dateValueLow)) {
4395       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4396       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4397     }
4398     elseif (!empty($dateValueHigh)) {
4399       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4400       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4401     }
4402
4403     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4404       $where[$grouping][] = $activePeriodClauses;
4405     }
4406   }
4407
4408   /**
4409    * Get start & end active period criteria
4410    *
4411    * @param $from
4412    * @param $to
4413    * @param $forceTableName
4414    *
4415    * @return string
4416    */
4417   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4418     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4419     if (!is_null($from) && !is_null($to)) {
4420       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4421                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4422                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4423                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4424                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4425                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4426     }
4427     elseif (!is_null($from)) {
4428       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4429                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4430                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4431     }
4432     elseif (!is_null($to)) {
4433       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4434                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4435                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4436     }
4437   }
4438
4439   /**
4440    * Default set of return properties.
4441    *
4442    * @param int $mode
4443    *
4444    * @return array
4445    *   derault return properties
4446    */
4447   public static function &defaultReturnProperties($mode = 1) {
4448     if (!isset(self::$_defaultReturnProperties)) {
4449       self::$_defaultReturnProperties = [];
4450     }
4451
4452     if (!isset(self::$_defaultReturnProperties[$mode])) {
4453       // add activity return properties
4454       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4455         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4456       }
4457       else {
4458         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4459       }
4460
4461       if (empty(self::$_defaultReturnProperties[$mode])) {
4462         self::$_defaultReturnProperties[$mode] = [
4463           'image_URL' => 1,
4464           'legal_identifier' => 1,
4465           'external_identifier' => 1,
4466           'contact_type' => 1,
4467           'contact_sub_type' => 1,
4468           'sort_name' => 1,
4469           'display_name' => 1,
4470           'preferred_mail_format' => 1,
4471           'nick_name' => 1,
4472           'first_name' => 1,
4473           'middle_name' => 1,
4474           'last_name' => 1,
4475           'prefix_id' => 1,
4476           'suffix_id' => 1,
4477           'formal_title' => 1,
4478           'communication_style_id' => 1,
4479           'birth_date' => 1,
4480           'gender_id' => 1,
4481           'street_address' => 1,
4482           'supplemental_address_1' => 1,
4483           'supplemental_address_2' => 1,
4484           'supplemental_address_3' => 1,
4485           'city' => 1,
4486           'postal_code' => 1,
4487           'postal_code_suffix' => 1,
4488           'state_province' => 1,
4489           'country' => 1,
4490           'world_region' => 1,
4491           'geo_code_1' => 1,
4492           'geo_code_2' => 1,
4493           'email' => 1,
4494           'on_hold' => 1,
4495           'phone' => 1,
4496           'im' => 1,
4497           'household_name' => 1,
4498           'organization_name' => 1,
4499           'deceased_date' => 1,
4500           'is_deceased' => 1,
4501           'job_title' => 1,
4502           'legal_name' => 1,
4503           'sic_code' => 1,
4504           'current_employer' => 1,
4505           // FIXME: should we use defaultHierReturnProperties() for the below?
4506           'do_not_email' => 1,
4507           'do_not_mail' => 1,
4508           'do_not_sms' => 1,
4509           'do_not_phone' => 1,
4510           'do_not_trade' => 1,
4511           'is_opt_out' => 1,
4512           'contact_is_deleted' => 1,
4513           'preferred_communication_method' => 1,
4514           'preferred_language' => 1,
4515         ];
4516       }
4517     }
4518     return self::$_defaultReturnProperties[$mode];
4519   }
4520
4521   /**
4522    * Get primary condition for a sql clause.
4523    *
4524    * @param int $value
4525    *
4526    * @return string|NULL
4527    */
4528   public static function getPrimaryCondition($value) {
4529     if (is_numeric($value)) {
4530       $value = (int ) $value;
4531       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4532     }
4533     return NULL;
4534   }
4535
4536   /**
4537    * Wrapper for a simple search query.
4538    *
4539    * @param array $params
4540    * @param array $returnProperties
4541    * @param bool $count
4542    *
4543    * @return string
4544    * @throws \CRM_Core_Exception
4545    */
4546   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4547     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4548     list($select, $from, $where, $having) = $query->query();
4549     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4550
4551     $query = "$select $from $where $groupBy $having";
4552     return $query;
4553   }
4554
4555   /**
4556    * These are stub comments as this function needs more explanation - particularly in terms of how it
4557    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4558    *
4559    * This function was originally written as a wrapper for the api query but is called from multiple places
4560    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4561    * but it is unclear as to whehter that is historical or there is a reason
4562    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4563    *
4564    * @param array $params
4565    * @param array $returnProperties
4566    * @param null $fields
4567    * @param string $sort
4568    * @param int $offset
4569    * @param int $row_count
4570    * @param bool $smartGroupCache
4571    *   ?? update smart group cache?.
4572    * @param bool $count
4573    *   Return count obnly.
4574    * @param bool $skipPermissions
4575    *   Should permissions be ignored or should the logged in user's permissions be applied.
4576    * @param int $mode
4577    *   This basically correlates to the component.
4578    * @param string $apiEntity
4579    *   The api entity being called.
4580    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4581    *
4582    * @param bool|null $primaryLocationOnly
4583    *
4584    * @return array
4585    * @throws \CRM_Core_Exception
4586    */
4587   public static function apiQuery(
4588     $params = NULL,
4589     $returnProperties = NULL,
4590     $fields = NULL,
4591     $sort = NULL,
4592     $offset = 0,
4593     $row_count = 25,
4594     $smartGroupCache = TRUE,
4595     $count = FALSE,
4596     $skipPermissions = TRUE,
4597     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4598     $apiEntity = NULL,
4599     $primaryLocationOnly = NULL
4600   ) {
4601
4602     $query = new CRM_Contact_BAO_Query(
4603       $params, $returnProperties,
4604       NULL, TRUE, FALSE, $mode,
4605       $skipPermissions,
4606       TRUE, $smartGroupCache,
4607       NULL, 'AND',
4608       $apiEntity, $primaryLocationOnly
4609     );
4610
4611     //this should add a check for view deleted if permissions are enabled
4612     if ($skipPermissions) {
4613       $query->_skipDeleteClause = TRUE;
4614     }
4615     $query->generatePermissionClause(FALSE, $count);
4616
4617     // note : this modifies _fromClause and _simpleFromClause
4618     $query->includePseudoFieldsJoin($sort);
4619
4620     list($select, $from, $where, $having) = $query->query($count);
4621
4622     if (!empty($query->_permissionWhereClause)) {
4623       if (!empty($query->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
4624         $from .= " $query->_permissionFromClause";
4625       }
4626       if (empty($where)) {
4627         $where = "WHERE $query->_permissionWhereClause";
4628       }
4629       else {
4630         $where = "$where AND $query->_permissionWhereClause";
4631       }
4632     }
4633
4634     $sql = "$select $from $where $having";
4635
4636     // add group by only when API action is not getcount
4637     //  otherwise query fetches incorrect count
4638     if ($query->_useGroupBy && !$count) {
4639       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4640     }
4641     if (!empty($sort)) {
4642       $sort = CRM_Utils_Type::escape($sort, 'String');
4643       $sql .= " ORDER BY $sort ";
4644     }
4645     if ($row_count > 0 && $offset >= 0) {
4646       $offset = CRM_Utils_Type::escape($offset, 'Int');
4647       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4648       $sql .= " LIMIT $offset, $row_count ";
4649     }
4650
4651     $dao = CRM_Core_DAO::executeQuery($sql);
4652
4653     // @todo derive this from the component class rather than hard-code two options.
4654     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4655
4656     $values = [];
4657     while ($dao->fetch()) {
4658       if ($count) {
4659         $noRows = $dao->rowCount;
4660         return [$noRows, NULL];
4661       }
4662       $val = $query->store($dao);
4663       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4664
4665       if (!empty($convertedVals)) {
4666         $val = array_replace_recursive($val, $convertedVals);
4667       }
4668       $values[$dao->$entityIDField] = $val;
4669     }
4670     return [$values];
4671   }
4672
4673   /**
4674    * Get the actual custom field name by stripping off the appended string.
4675    *
4676    * The string could be _relative, _from, or _to
4677    *
4678    * @todo use metadata rather than convention to do this.
4679    *
4680    * @param string $parameterName
4681    *   The name of the parameter submitted to the form.
4682    *   e.g
4683    *   custom_3_relative
4684    *   custom_3_from
4685    *
4686    * @return string
4687    */
4688   public static function getCustomFieldName($parameterName) {
4689     if (substr($parameterName, -5, 5) == '_from') {
4690       return substr($parameterName, 0, strpos($parameterName, '_from'));
4691     }
4692     if (substr($parameterName, -9, 9) == '_relative') {
4693       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4694     }
4695     if (substr($parameterName, -3, 3) == '_to') {
4696       return substr($parameterName, 0, strpos($parameterName, '_to'));
4697     }
4698   }
4699
4700   /**
4701    * Convert submitted values for relative custom fields to query object format.
4702    *
4703    * The query will support the sqlOperator format so convert to that format.
4704    *
4705    * @param array $formValues
4706    *   Submitted values.
4707    * @param array $params
4708    *   Converted parameters for the query object.
4709    * @param string $values
4710    *   Submitted value.
4711    * @param string $fieldName
4712    *   Submitted field name. (Matches form field not DB field.)
4713    */
4714   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4715     if (empty($values)) {
4716       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4717       // this function gets called again after they fields have been converted which can get ugly.
4718       return;
4719     }
4720     $customFieldName = self::getCustomFieldName($fieldName);
4721
4722     if (substr($fieldName, -9, 9) == '_relative') {
4723       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4724     }
4725     else {
4726       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4727         // Both to & from are set. We only need to acton one, choosing from.
4728         return;
4729       }
4730
4731       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4732       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4733
4734       if (self::isCustomDateField($customFieldName)) {
4735         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4736       }
4737     }
4738
4739     if ($from) {
4740       if ($to) {
4741         $relativeFunction = ['BETWEEN' => [$from, $to]];
4742       }
4743       else {
4744         $relativeFunction = ['>=' => $from];
4745       }
4746     }
4747     else {
4748       $relativeFunction = ['<=' => $to];
4749     }
4750     $params[] = [
4751       $customFieldName,
4752       '=',
4753       $relativeFunction,
4754       0,
4755       0,
4756     ];
4757   }
4758
4759   /**
4760    * Are we dealing with custom field of type date.
4761    *
4762    * @param $fieldName
4763    *
4764    * @return bool
4765    * @throws Exception
4766    */
4767   public static function isCustomDateField($fieldName) {
4768     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4769       return FALSE;
4770     }
4771     try {
4772       $customFieldData = CRM_Core_BAO_CustomField::getFieldObject($customFieldID);
4773       $customFieldDataType = $customFieldData->data_type;
4774       if ('Date' == $customFieldDataType) {
4775         return TRUE;
4776       }
4777     }
4778     catch (Exception $e) {
4779     }
4780     return FALSE;
4781   }
4782
4783   /**
4784    * Has this field already been reformatting to Query object syntax.
4785    *
4786    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4787    * to come with associated double queries & is possibly damaging performance.
4788    *
4789    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4790    * them as they are.
4791    *
4792    * @param mixed $values
4793    *   Value in formValues for the field.
4794    *
4795    * @return bool;
4796    */
4797   public static function isAlreadyProcessedForQueryFormat($values) {
4798     if (!is_array($values)) {
4799       return FALSE;
4800     }
4801     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4802       return FALSE;
4803     }
4804     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4805   }
4806
4807   /**
4808    * If the state and country are passed remove state.
4809    *
4810    * Country is implicit from the state, but including both results in
4811    * a poor query as there is no combined index on state AND country.
4812    *
4813    * @see https://issues.civicrm.org/jira/browse/CRM-18125
4814    *
4815    * @param array $formValues
4816    */
4817   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4818     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4819       // The use of array map sanitises the data by ensuring we are dealing with integers.
4820       $states = implode(', ', array_map('intval', $formValues['state_province']));
4821       $countryList = CRM_Core_DAO::singleValueQuery(
4822         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4823       );
4824       if ($countryList == $formValues['country']) {
4825         unset($formValues['country']);
4826       }
4827     }
4828   }
4829
4830   /**
4831    * For some special cases, grouping by subset of select fields becomes mandatory.
4832    * Hence, full_group_by mode is handled by appending any_value
4833    * keyword to select fields not present in groupBy
4834    *
4835    * @param array $selectClauses
4836    * @param array $groupBy - Columns already included in GROUP By clause.
4837    * @param string $aggregateFunction
4838    *
4839    * @return string
4840    */
4841   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4842     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4843       $groupBy = array_map('trim', (array) $groupBy);
4844       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4845       foreach ($selectClauses as $key => &$val) {
4846         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4847         // append ANY_VALUE() keyword
4848         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4849           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4850             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4851             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4852         }
4853       }
4854     }
4855
4856     return "SELECT " . implode(', ', $selectClauses) . " ";
4857   }
4858
4859   /**
4860    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4861    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4862    * keyword to select fields not present in groupBy
4863    *
4864    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4865    * @param array $orderBys - ORDER BY sub-clauses
4866    *
4867    */
4868   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4869     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4870       foreach ($orderBys as $orderBy) {
4871         // remove sort syntax from ORDER BY clauses if present
4872         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4873         // if ORDER BY column is not present in GROUP BY then append it to end
4874         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4875           $groupBy .= ", {$orderBy}";
4876         }
4877       }
4878     }
4879   }
4880
4881   /**
4882    * Include select columns in groupBy clause.
4883    *
4884    * @param array $selectClauses
4885    * @param array|string|null $groupBy - Columns already included in GROUP By clause.
4886    *
4887    * @return string
4888    */
4889   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4890     $groupBy = (array) $groupBy;
4891     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4892
4893     //return if ONLY_FULL_GROUP_BY is not enabled.
4894     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4895       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4896       foreach ($selectClauses as $key => $val) {
4897         $aliasArray = preg_split('/ as /i', $val);
4898         // if more than 1 alias we need to split by ','.
4899         if (count($aliasArray) > 2) {
4900           $aliasArray = preg_split('/,/', $val);
4901           foreach ($aliasArray as $key => $value) {
4902             $alias = current(preg_split('/ as /i', $value));
4903             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4904               $groupBy[] = $alias;
4905             }
4906           }
4907         }
4908         else {
4909           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4910           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4911           $tableName = current(explode('.', $selectColumn));
4912           $primaryKey = "{$tableName}.id";
4913           // exclude columns which are already included in groupBy and aggregate functions from select
4914           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4915           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4916             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4917               $groupBy[] = $alias;
4918             }
4919             else {
4920               $groupBy[] = $selectColumn;
4921             }
4922           }
4923         }
4924       }
4925     }
4926
4927     if (!empty($groupBy)) {
4928       return " GROUP BY " . implode(', ', $groupBy);
4929     }
4930     return '';
4931   }
4932
4933   /**
4934    * Create and query the db for an contact search.
4935    *
4936    * @param int $offset
4937    *   The offset for the query.
4938    * @param int $rowCount
4939    *   The number of rows to return.
4940    * @param string|CRM_Utils_Sort $sort
4941    *   The order by string.
4942    * @param bool $count
4943    *   Is this a count only query ?.
4944    * @param bool $includeContactIds
4945    *   Should we include contact ids?.
4946    * @param bool $sortByChar
4947    *   If true returns the distinct array of first characters for search results.
4948    * @param bool $groupContacts
4949    *   If true, return only the contact ids.
4950    * @param bool $returnQuery
4951    *   Should we return the query as a string.
4952    * @param string $additionalWhereClause
4953    *   If the caller wants to further restrict the search (used for components).
4954    * @param null $sortOrder
4955    * @param string $additionalFromClause
4956    *   Should be clause with proper joins, effective to reduce where clause load.
4957    *
4958    * @param bool $skipOrderAndLimit
4959    *
4960    * @return string|null|CRM_Core_DAO
4961    */
4962   public function searchQuery(
4963     $offset = 0, $rowCount = 0, $sort = NULL,
4964     $count = FALSE, $includeContactIds = FALSE,
4965     $sortByChar = FALSE, $groupContacts = FALSE,
4966     $returnQuery = FALSE,
4967     $additionalWhereClause = NULL, $sortOrder = NULL,
4968     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4969   ) {
4970
4971     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4972
4973     if ($returnQuery) {
4974       return $query;
4975     }
4976     if ($count) {
4977       return CRM_Core_DAO::singleValueQuery($query);
4978     }
4979
4980     $dao = CRM_Core_DAO::executeQuery($query);
4981
4982     // We can always call this - it will only re-enable if it was originally enabled.
4983     CRM_Core_DAO::reenableFullGroupByMode();
4984
4985     if ($groupContacts) {
4986       $ids = [];
4987       while ($dao->fetch()) {
4988         $ids[] = $dao->id;
4989       }
4990       return implode(',', $ids);
4991     }
4992
4993     return $dao;
4994   }
4995
4996   /**
4997    * Create and query the db for the list of all first letters used by contacts
4998    *
4999    * @return CRM_Core_DAO
5000    */
5001   public function alphabetQuery() {
5002     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
5003     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
5004       {$sqlParts['from']}
5005       {$sqlParts['where']}";
5006     $dao = CRM_Core_DAO::executeQuery($query);
5007     return $dao;
5008   }
5009
5010   /**
5011    * Fetch a list of contacts for displaying a search results page
5012    *
5013    * @param array $cids
5014    *   List of contact IDs
5015    * @param bool $includeContactIds
5016    * @return CRM_Core_DAO
5017    */
5018   public function getCachedContacts($cids, $includeContactIds) {
5019     CRM_Core_DAO::disableFullGroupByMode();
5020     CRM_Utils_Type::validateAll($cids, 'Positive');
5021     $this->_includeContactIds = $includeContactIds;
5022     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5023     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5024     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5025     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5026     $order = 'ORDER BY _wgt';
5027     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5028     $limit = '';
5029     $query = "$select $from $where $groupBy $order $limit";
5030
5031     $result = CRM_Core_DAO::executeQuery($query);
5032     CRM_Core_DAO::reenableFullGroupByMode();
5033     return $result;
5034   }
5035
5036   /**
5037    * Construct a SQL CASE expression.
5038    *
5039    * @param string $idCol
5040    *   The name of a column with ID's (eg 'contact_a.id').
5041    * @param array $cids
5042    *   Array(int $weight => int $id).
5043    * @return string
5044    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5045    */
5046   private function createSqlCase($idCol, $cids) {
5047     $buf = "CASE\n";
5048     foreach ($cids as $weight => $cid) {
5049       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5050     }
5051     $buf .= "END\n";
5052     return $buf;
5053   }
5054
5055   /**
5056    * Populate $this->_permissionWhereClause with permission related clause and update other
5057    * query related properties.
5058    *
5059    * Function calls ACL permission class and hooks to filter the query appropriately
5060    *
5061    * Note that these 2 params were in the code when extracted from another function
5062    * and a second round extraction would be to make them properties of the class
5063    *
5064    * @param bool $onlyDeleted
5065    *   Only get deleted contacts.
5066    * @param bool $count
5067    *   Return Count only.
5068    */
5069   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5070     if (!$this->_skipPermission) {
5071       $permissionClauses = CRM_Contact_BAO_Contact_Permission::cacheClause();
5072       $this->_permissionWhereClause = $permissionClauses[1];
5073       $this->_permissionFromClause = $permissionClauses[0];
5074
5075       if (CRM_Core_Permission::check('access deleted contacts')) {
5076         if (!$onlyDeleted) {
5077           $this->_permissionWhereClause .= ' AND (contact_a.is_deleted = 0)';
5078         }
5079         else {
5080           $this->_permissionWhereClause .= " AND (contact_a.is_deleted) ";
5081         }
5082       }
5083
5084       if (isset($this->_tables['civicrm_activity'])) {
5085         $bao = new CRM_Activity_BAO_Activity();
5086         $clauses = $subclauses = [];
5087         foreach ((array) $bao->addSelectWhereClause() as $field => $vals) {
5088           if ($vals && $field !== 'id') {
5089             $clauses[] = $bao->tableName() . ".$field " . $vals;
5090           }
5091           elseif ($vals) {
5092             $subclauses[] = "$field " . implode(" AND $field ", (array) $vals);
5093           }
5094         }
5095         if ($subclauses) {
5096           $clauses[] = $bao->tableName() . '.`id` IN (SELECT `id` FROM `' . $bao->tableName() . '` WHERE ' . implode(' AND ', $subclauses) . ')';
5097         }
5098         if (!empty($clauses) && $this->_permissionWhereClause) {
5099           $this->_permissionWhereClause .= ' AND (' . implode(' AND ', $clauses) . ')';
5100         }
5101         elseif (!empty($clauses)) {
5102           $this->_permissionWhereClause .= '(' . implode(' AND ', $clauses) . ')';
5103         }
5104       }
5105     }
5106     else {
5107       // add delete clause if needed even if we are skipping permission
5108       // CRM-7639
5109       if (!$this->_skipDeleteClause) {
5110         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5111           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5112         }
5113         else {
5114           // CRM-6181
5115           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5116         }
5117       }
5118     }
5119   }
5120
5121   /**
5122    * @param $val
5123    */
5124   public function setSkipPermission($val) {
5125     $this->_skipPermission = $val;
5126   }
5127
5128   /**
5129    * @param null $context
5130    *
5131    * @return array
5132    * @throws \CRM_Core_Exception
5133    */
5134   public function summaryContribution($context = NULL) {
5135     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5136     if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5137       $from .= " $this->_permissionFromClause";
5138     }
5139     if ($this->_permissionWhereClause) {
5140       $where .= " AND " . $this->_permissionWhereClause;
5141     }
5142     if ($context == 'search') {
5143       $where .= " AND contact_a.is_deleted = 0 ";
5144     }
5145
5146     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5147
5148     $summary = ['total' => [], 'soft_credit' => ['count' => 0, 'avg' => 0, 'amount' => 0]];
5149     $this->addBasicStatsToSummary($summary, $where, $from);
5150
5151     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5152       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5153     }
5154
5155     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5156
5157     return $summary;
5158   }
5159
5160   /**
5161    * Append financial ACL limits to the query from & where clauses, if applicable.
5162    *
5163    * @param string $where
5164    * @param string $from
5165    */
5166   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5167     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5168       return;
5169     }
5170     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5171     if (!empty($financialTypes)) {
5172       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5173       $from .= " LEFT JOIN civicrm_line_item li
5174                   ON civicrm_contribution.id = li.contribution_id AND
5175                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5176     }
5177     else {
5178       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5179     }
5180   }
5181
5182   /**
5183    * Getter for the qill object.
5184    *
5185    * @return array
5186    */
5187   public function qill() {
5188     return $this->_qill;
5189   }
5190
5191   /**
5192    * Default set of return default hier return properties.
5193    *
5194    * @return array
5195    */
5196   public static function &defaultHierReturnProperties() {
5197     if (!isset(self::$_defaultHierReturnProperties)) {
5198       self::$_defaultHierReturnProperties = [
5199         'home_URL' => 1,
5200         'image_URL' => 1,
5201         'legal_identifier' => 1,
5202         'external_identifier' => 1,
5203         'contact_type' => 1,
5204         'contact_sub_type' => 1,
5205         'sort_name' => 1,
5206         'display_name' => 1,
5207         'nick_name' => 1,
5208         'first_name' => 1,
5209         'middle_name' => 1,
5210         'last_name' => 1,
5211         'prefix_id' => 1,
5212         'suffix_id' => 1,
5213         'formal_title' => 1,
5214         'communication_style_id' => 1,
5215         'email_greeting' => 1,
5216         'postal_greeting' => 1,
5217         'addressee' => 1,
5218         'birth_date' => 1,
5219         'gender_id' => 1,
5220         'preferred_communication_method' => 1,
5221         'do_not_phone' => 1,
5222         'do_not_email' => 1,
5223         'do_not_mail' => 1,
5224         'do_not_sms' => 1,
5225         'do_not_trade' => 1,
5226         'location' => [
5227           '1' => [
5228             'location_type' => 1,
5229             'street_address' => 1,
5230             'city' => 1,
5231             'state_province' => 1,
5232             'postal_code' => 1,
5233             'postal_code_suffix' => 1,
5234             'country' => 1,
5235             'phone-Phone' => 1,
5236             'phone-Mobile' => 1,
5237             'phone-Fax' => 1,
5238             'phone-1' => 1,
5239             'phone-2' => 1,
5240             'phone-3' => 1,
5241             'im-1' => 1,
5242             'im-2' => 1,
5243             'im-3' => 1,
5244             'email-1' => 1,
5245             'email-2' => 1,
5246             'email-3' => 1,
5247           ],
5248           '2' => [
5249             'location_type' => 1,
5250             'street_address' => 1,
5251             'city' => 1,
5252             'state_province' => 1,
5253             'postal_code' => 1,
5254             'postal_code_suffix' => 1,
5255             'country' => 1,
5256             'phone-Phone' => 1,
5257             'phone-Mobile' => 1,
5258             'phone-1' => 1,
5259             'phone-2' => 1,
5260             'phone-3' => 1,
5261             'im-1' => 1,
5262             'im-2' => 1,
5263             'im-3' => 1,
5264             'email-1' => 1,
5265             'email-2' => 1,
5266             'email-3' => 1,
5267           ],
5268         ],
5269       ];
5270     }
5271     return self::$_defaultHierReturnProperties;
5272   }
5273
5274   /**
5275    * Build query for a date field.
5276    *
5277    * @param array $values
5278    * @param string $tableName
5279    * @param string $fieldName
5280    * @param string $dbFieldName
5281    * @param string $fieldTitle
5282    * @param bool $appendTimeStamp
5283    * @param string $dateFormat
5284    * @param string|null $highDBFieldName
5285    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5286    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5287    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5288    *   date & less than the end date.
5289    *
5290    * @throws \CRM_Core_Exception
5291    */
5292   public function dateQueryBuilder(
5293     $values, $tableName, $fieldName,
5294     $dbFieldName, $fieldTitle,
5295     $appendTimeStamp = TRUE,
5296     $dateFormat = 'YmdHis',
5297     $highDBFieldName = NULL
5298   ) {
5299     // @todo - remove dateFormat - pretty sure it's never passed in...
5300     list($name, $op, $value, $grouping, $wildcard) = $values;
5301     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5302       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5303       return;
5304     }
5305     if ($tableName === 'civicrm_contact') {
5306       // Special handling for contact table as it has a known alias in advanced search.
5307       $tableName = 'contact_a';
5308     }
5309     if ($name === "{$fieldName}_low" ||
5310       $name === "{$fieldName}_high"
5311     ) {
5312       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5313         return;
5314       }
5315       $this->_rangeCache[$fieldName] = 1;
5316
5317       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5318
5319       if ($name == $fieldName . '_low') {
5320         $firstOP = '>=';
5321         $firstPhrase = ts('greater than or equal to');
5322         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5323
5324         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5325         if (!empty($secondValues) && $secondValues[2]) {
5326           $secondOP = '<=';
5327           $secondPhrase = ts('less than or equal to');
5328           $secondValue = $secondValues[2];
5329
5330           if ($appendTimeStamp && strlen($secondValue) == 10) {
5331             $secondValue .= ' 23:59:59';
5332           }
5333           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5334         }
5335       }
5336       elseif ($name == $fieldName . '_high') {
5337         $firstOP = '<=';
5338         $firstPhrase = ts('less than or equal to');
5339
5340         if ($appendTimeStamp && strlen($value) == 10) {
5341           $value .= ' 23:59:59';
5342         }
5343         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5344
5345         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5346         if (!empty($secondValues) && $secondValues[2]) {
5347           $secondOP = '>=';
5348           $secondPhrase = ts('greater than or equal to');
5349           $secondValue = $secondValues[2];
5350           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5351         }
5352       }
5353
5354       if (!$appendTimeStamp) {
5355         $firstDate = substr($firstDate, 0, 8);
5356       }
5357       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5358
5359       if ($secondDate) {
5360         if (!$appendTimeStamp) {
5361           $secondDate = substr($secondDate, 0, 8);
5362         }
5363         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5364       }
5365
5366       if ($secondDate) {
5367         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5368         $this->_where[$grouping][] = "
5369 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5370 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5371 ";
5372         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5373       }
5374       else {
5375         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5376         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5377       }
5378     }
5379
5380     if ($name == $fieldName) {
5381       //In Get API, for operators other then '=' the $value is in array(op => value) format
5382       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5383         $op = key($value);
5384         $value = $value[$op];
5385       }
5386
5387       $date = $format = NULL;
5388       if (strstr($op, 'IN')) {
5389         $format = [];
5390         foreach ($value as &$date) {
5391           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5392           if (!$appendTimeStamp) {
5393             $date = substr($date, 0, 8);
5394           }
5395           $format[] = CRM_Utils_Date::customFormat($date);
5396         }
5397         $date = "('" . implode("','", $value) . "')";
5398         $format = implode(', ', $format);
5399       }
5400       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5401         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5402         if (!$appendTimeStamp) {
5403           $date = substr($date, 0, 8);
5404         }
5405         $format = CRM_Utils_Date::customFormat($date);
5406         $date = "'$date'";
5407       }
5408
5409       if ($date) {
5410         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5411       }
5412       else {
5413         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5414       }
5415
5416       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5417       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5418     }
5419
5420     // Ensure the tables are set, but don't whomp anything.
5421     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5422     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5423   }
5424
5425   /**
5426    * @param $values
5427    * @param string $tableName
5428    * @param string $fieldName
5429    * @param string $dbFieldName
5430    * @param $fieldTitle
5431    * @param null $options
5432    */
5433   public function numberRangeBuilder(
5434     &$values,
5435     $tableName, $fieldName,
5436     $dbFieldName, $fieldTitle,
5437     $options = NULL
5438   ) {
5439     list($name, $op, $value, $grouping, $wildcard) = $values;
5440
5441     if ($name == "{$fieldName}_low" ||
5442       $name == "{$fieldName}_high"
5443     ) {
5444       if (isset($this->_rangeCache[$fieldName])) {
5445         return;
5446       }
5447       $this->_rangeCache[$fieldName] = 1;
5448
5449       $secondOP = $secondPhrase = $secondValue = NULL;
5450
5451       if ($name == "{$fieldName}_low") {
5452         $firstOP = '>=';
5453         $firstPhrase = ts('greater than');
5454
5455         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5456         if (!empty($secondValues)) {
5457           $secondOP = '<=';
5458           $secondPhrase = ts('less than');
5459           $secondValue = $secondValues[2];
5460         }
5461       }
5462       else {
5463         $firstOP = '<=';
5464         $firstPhrase = ts('less than');
5465
5466         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5467         if (!empty($secondValues)) {
5468           $secondOP = '>=';
5469           $secondPhrase = ts('greater than');
5470           $secondValue = $secondValues[2];
5471         }
5472       }
5473
5474       if ($secondOP) {
5475         $this->_where[$grouping][] = "
5476 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5477 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5478 ";
5479         $displayValue = $options ? $options[$value] : $value;
5480         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5481
5482         $this->_qill[$grouping][]
5483           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5484       }
5485       else {
5486         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5487         $displayValue = $options ? $options[$value] : $value;
5488         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5489       }
5490       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5491
5492       return;
5493     }
5494
5495     if ($name == $fieldName) {
5496       $op = '=';
5497       $phrase = '=';
5498
5499       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5500
5501       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5502       $displayValue = $options ? $options[$value] : $value;
5503       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5504     }
5505   }
5506
5507   /**
5508    * @param $values
5509    * @param string $tableName
5510    * @param string $fieldName
5511    * @param string $dbFieldName
5512    * @param $fieldTitle
5513    * @param null $options
5514    */
5515   public function ageRangeQueryBuilder(
5516     &$values,
5517     $tableName, $fieldName,
5518     $dbFieldName, $fieldTitle,
5519     $options = NULL
5520   ) {
5521     list($name, $op, $value, $grouping, $wildcard) = $values;
5522
5523     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5524     // will be treated as current day
5525     $asofDate = NULL;
5526     if ($asofDateValues) {
5527       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5528       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5529       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5530     }
5531
5532     if ($name == "{$fieldName}_low" ||
5533       $name == "{$fieldName}_high"
5534     ) {
5535       if (isset($this->_rangeCache[$fieldName])) {
5536         return;
5537       }
5538       $this->_rangeCache[$fieldName] = 1;
5539
5540       $secondOP = $secondPhrase = $secondValue = NULL;
5541
5542       if ($name == "{$fieldName}_low") {
5543         $firstPhrase = ts('greater than or equal to');
5544         // NB: age > X means date of birth < Y
5545         $firstOP = '<=';
5546         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5547
5548         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5549         if (!empty($secondValues)) {
5550           $secondOP = '>=';
5551           $secondPhrase = ts('less than or equal to');
5552           $secondValue = $secondValues[2];
5553           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5554         }
5555       }
5556       else {
5557         $firstOP = '>=';
5558         $firstPhrase = ts('less than or equal to');
5559         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5560
5561         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5562         if (!empty($secondValues)) {
5563           $secondOP = '<=';
5564           $secondPhrase = ts('greater than or equal to');
5565           $secondValue = $secondValues[2];
5566           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5567         }
5568       }
5569
5570       if ($secondOP) {
5571         $this->_where[$grouping][] = "
5572 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5573 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5574 ";
5575         $displayValue = $options ? $options[$value] : $value;
5576         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5577
5578         $this->_qill[$grouping][]
5579           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5580       }
5581       else {
5582         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5583         $displayValue = $options ? $options[$value] : $value;
5584         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5585       }
5586       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5587       return;
5588     }
5589   }
5590
5591   /**
5592    * Calculate date from age.
5593    *
5594    * @param string $asofDate
5595    * @param int $age
5596    * @param string $type
5597    *
5598    * @return string
5599    * @throws \Exception
5600    */
5601   public static function calcDateFromAge($asofDate, $age, $type) {
5602     $date = new DateTime($asofDate);
5603     if ($type == "min") {
5604       // minimum age is $age: dob <= date - age "235959"
5605       $date->sub(new DateInterval("P" . $age . "Y"));
5606       return $date->format('Ymd') . "235959";
5607     }
5608     else {
5609       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5610       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5611       return $date->format('Ymd') . "000000";
5612     }
5613   }
5614
5615   /**
5616    * Given the field name, operator, value & its data type
5617    * builds the where Clause for the query
5618    * used for handling 'IS NULL'/'IS NOT NULL' operators
5619    *
5620    * @param string $field
5621    *   Fieldname.
5622    * @param string $op
5623    *   Operator.
5624    * @param string $value
5625    *   Value.
5626    * @param string $dataType
5627    *   Data type of the field.
5628    *
5629    * @return string
5630    *   Where clause for the query.
5631    * @throws \CRM_Core_Exception
5632    */
5633   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5634     $op = trim($op);
5635     $clause = "$field $op";
5636
5637     switch ($op) {
5638       case 'IS NULL':
5639       case 'IS NOT NULL':
5640         return $clause;
5641
5642       case 'IS EMPTY':
5643         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5644         return $clause;
5645
5646       case 'IS NOT EMPTY':
5647         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5648         return $clause;
5649
5650       case 'RLIKE':
5651         return " CAST({$field} AS BINARY) RLIKE BINARY '{$value}' ";
5652
5653       case 'IN':
5654       case 'NOT IN':
5655         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5656         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5657           $value = [$op => (array) $value];
5658         }
5659
5660       default:
5661         if (empty($dataType) || $dataType == 'Date') {
5662           $dataType = 'String';
5663         }
5664         if (is_array($value)) {
5665           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5666           // widely used and consistent across the codebase
5667           // adding this here won't accept the search functions which don't submit an array
5668           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5669
5670             return $queryString;
5671           }
5672           if (!empty($value[0]) && $op === 'BETWEEN') {
5673             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5674             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5675               return $queryString;
5676             }
5677           }
5678           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5679         }
5680         $emojiWhere = CRM_Utils_SQL::handleEmojiInQuery($value);
5681         if ($emojiWhere === '0 = 1') {
5682           $value = $emojiWhere;
5683         }
5684         $value = CRM_Utils_Type::escape($value, $dataType);
5685         // if we don't have a dataType we should assume
5686         if ($dataType == 'String' || $dataType == 'Text') {
5687           $value = "'" . $value . "'";
5688         }
5689         return "$clause $value";
5690     }
5691   }
5692
5693   /**
5694    * @param bool $reset
5695    *
5696    * @return array
5697    */
5698   public function openedSearchPanes($reset = FALSE) {
5699     if (!$reset || empty($this->_whereTables)) {
5700       return self::$_openedPanes;
5701     }
5702
5703     // pane name to table mapper
5704     $panesMapper = [
5705       ts('Contributions') => 'civicrm_contribution',
5706       ts('Memberships') => 'civicrm_membership',
5707       ts('Events') => 'civicrm_participant',
5708       ts('Relationships') => 'civicrm_relationship',
5709       ts('Activities') => 'civicrm_activity',
5710       ts('Pledges') => 'civicrm_pledge',
5711       ts('Cases') => 'civicrm_case',
5712       ts('Address Fields') => 'civicrm_address',
5713       ts('Notes') => 'civicrm_note',
5714       ts('Change Log') => 'civicrm_log',
5715       ts('Mailings') => 'civicrm_mailing',
5716     ];
5717     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5718
5719     foreach (array_keys($this->_whereTables) as $table) {
5720       if ($panName = array_search($table, $panesMapper)) {
5721         self::$_openedPanes[$panName] = TRUE;
5722       }
5723     }
5724
5725     return self::$_openedPanes;
5726   }
5727
5728   /**
5729    * @param $operator
5730    */
5731   public function setOperator($operator) {
5732     $validOperators = ['AND', 'OR'];
5733     if (!in_array($operator, $validOperators)) {
5734       $operator = 'AND';
5735     }
5736     $this->_operator = $operator;
5737   }
5738
5739   /**
5740    * @return string
5741    */
5742   public function getOperator() {
5743     return $this->_operator;
5744   }
5745
5746   /**
5747    * @param $from
5748    * @param $where
5749    * @param $having
5750    */
5751   public function filterRelatedContacts(&$from, &$where, &$having) {
5752     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5753       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5754     }
5755     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5756     // since there only can be one instance of this filter in every query
5757     // skip if filter has already applied
5758     foreach ($_rTempCache as $acache) {
5759       foreach ($acache['queries'] as $aqcache) {
5760         if (strpos($from, $aqcache['from']) !== FALSE) {
5761           $having = NULL;
5762           return;
5763         }
5764       }
5765     }
5766     $arg_sig = sha1("$from $where $having");
5767     if (isset($_rTempCache[$arg_sig])) {
5768       $cache = $_rTempCache[$arg_sig];
5769     }
5770     else {
5771       // create temp table with contact ids
5772
5773       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5774
5775       $sql = "
5776 REPLACE INTO $tableName ( contact_id )
5777 SELECT contact_a.id
5778        $from
5779        $where
5780        $having
5781 ";
5782       CRM_Core_DAO::executeQuery($sql);
5783
5784       $cache = ['tableName' => $tableName, 'queries' => []];
5785       $_rTempCache[$arg_sig] = $cache;
5786     }
5787     // upsert the query depending on relationship type
5788     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5789       $qcache = $cache['queries'][$this->_displayRelationshipType];
5790     }
5791     else {
5792       $tableName = $cache['tableName'];
5793       $qcache = [
5794         "from" => "",
5795         "where" => "",
5796       ];
5797       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5798       if (is_numeric($this->_displayRelationshipType)) {
5799         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5800         $qcache['from'] = "
5801 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5802 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5803 ";
5804         $qcache['where'] = "
5805 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5806 AND   displayRelType.is_active = 1
5807 ";
5808       }
5809       else {
5810         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5811         if ($dirOne == 'a') {
5812           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5813           $qcache['from'] .= "
5814 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5815 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5816 ";
5817         }
5818         else {
5819           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5820           $qcache['from'] .= "
5821 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5822 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5823 ";
5824         }
5825         $qcache['where'] = "
5826 WHERE displayRelType.relationship_type_id = $relType
5827 AND   displayRelType.is_active = 1
5828 ";
5829       }
5830       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5831       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5832     }
5833     $qillMessage = ts('Contacts with a Relationship Type of: ');
5834     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5835     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5836       $this->_qill[0][] = $iqill;
5837     }
5838     if (strpos($from, $qcache['from']) === FALSE) {
5839       if (strpos($from, "INNER JOIN") !== FALSE) {
5840         // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5841         // this happens when we have an event_type in the quert (CRM-7969)
5842         $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5843         // Make sure the relationship join right after the FROM and other joins afterwards.
5844         // This gives us the possibility to change the join on civicrm case.
5845         $from = preg_replace("/LEFT JOIN/", $qcache['from'] . " LEFT JOIN", $from, 1);
5846       }
5847       else {
5848         $from .= $qcache['from'];
5849       }
5850       $where = $qcache['where'];
5851       if (!empty($this->_tables['civicrm_case'])) {
5852         // Change the join on CiviCRM case so that it joins on the right contac from the relationship.
5853         $from = str_replace("ON civicrm_case_contact.contact_id = contact_a.id", "ON civicrm_case_contact.contact_id = transform_temp.contact_id", $from);
5854         $where .= " AND displayRelType.case_id = civicrm_case_contact.case_id ";
5855       }
5856       if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5857         $from .= " $this->_permissionFromClause";
5858       }
5859       if (!empty($this->_permissionWhereClause)) {
5860         $where .= "AND $this->_permissionWhereClause";
5861       }
5862     }
5863
5864     $having = NULL;
5865   }
5866
5867   /**
5868    * See CRM-19811 for why this is database hurty without apparent benefit.
5869    *
5870    * @param $op
5871    *
5872    * @return bool
5873    */
5874   public static function caseImportant($op) {
5875     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5876   }
5877
5878   /**
5879    * @param $returnProperties
5880    * @param $prefix
5881    *
5882    * @return bool
5883    */
5884   public static function componentPresent(&$returnProperties, $prefix) {
5885     foreach ($returnProperties as $name => $dontCare) {
5886       if (substr($name, 0, strlen($prefix)) == $prefix) {
5887         return TRUE;
5888       }
5889     }
5890     return FALSE;
5891   }
5892
5893   /**
5894    * Builds the necessary structures for all fields that are similar to option value look-ups.
5895    *
5896    * @param string $name
5897    *   the name of the field.
5898    * @param string $op
5899    *   the sql operator, this function should handle ALL SQL operators.
5900    * @param string $value
5901    *   depends on the operator and who's calling the query builder.
5902    * @param int $grouping
5903    *   the index where to place the where clause.
5904    * @param string $daoName
5905    *   DAO Name.
5906    * @param array $field
5907    *   an array that contains various properties of the field identified by $name.
5908    * @param string $label
5909    *   The label for this field element.
5910    * @param string $dataType
5911    *
5912    * @throws \CRM_Core_Exception
5913    */
5914   public function optionValueQuery(
5915     $name,
5916     $op,
5917     $value,
5918     $grouping,
5919     $daoName,
5920     $field,
5921     $label,
5922     $dataType = 'String'
5923   ) {
5924
5925     $pseudoFields = [
5926       'email_greeting',
5927       'postal_greeting',
5928       'addressee',
5929     ];
5930
5931     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5932     if ($tableName == 'civicrm_contact') {
5933       $wc = "contact_a.$fieldName";
5934     }
5935     else {
5936       // Special handling for on_hold, so that we actually use the 'where'
5937       // property in order to limit the query by the on_hold status of the email,
5938       // instead of using email.id which would be nonsensical.
5939       if ($field['name'] === 'on_hold') {
5940         $wc = $field['where'];
5941       }
5942       else {
5943         $wc = "$tableName.id";
5944       }
5945     }
5946
5947     if (in_array($name, $pseudoFields)) {
5948       $wc = "contact_a.{$name}_id";
5949       $dataType = 'Positive';
5950       $value = (!$value) ? 0 : $value;
5951     }
5952     if ($name == "world_region") {
5953       $field['name'] = $name;
5954     }
5955
5956     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5957     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5958     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5959   }
5960
5961   /**
5962    * Check and explode a user defined numeric string into an array
5963    * this was the protocol used by search builder in the old old days before we had
5964    * super nice js widgets to do the hard work
5965    *
5966    * @param string $string
5967    * @param string $dataType
5968    *   The dataType we should check for the values, default integer.
5969    *
5970    * @return bool|array
5971    *   false if string does not match the pattern
5972    *   array of numeric values if string does match the pattern
5973    */
5974   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5975     $string = trim($string);
5976     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5977       return FALSE;
5978     }
5979
5980     $string = substr($string, 1, -1);
5981     $values = explode(',', $string);
5982     if (empty($values)) {
5983       return FALSE;
5984     }
5985
5986     $returnValues = [];
5987     foreach ($values as $v) {
5988       if ($dataType == 'Integer' && !is_numeric($v)) {
5989         return FALSE;
5990       }
5991       elseif ($dataType == 'String' && !is_string($v)) {
5992         return FALSE;
5993       }
5994       $returnValues[] = trim($v);
5995     }
5996
5997     if (empty($returnValues)) {
5998       return FALSE;
5999     }
6000
6001     return $returnValues;
6002   }
6003
6004   /**
6005    * Convert the pseudo constants id's to their names
6006    *
6007    * @param CRM_Core_DAO $dao
6008    * @param bool $return
6009    * @param bool $usedForAPI
6010    *
6011    * @return array|NULL
6012    */
6013   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
6014     if (empty($this->_pseudoConstantsSelect)) {
6015       return NULL;
6016     }
6017     $values = [];
6018     foreach ($this->_pseudoConstantsSelect as $key => $value) {
6019       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
6020         continue;
6021       }
6022
6023       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6024         $val = $dao->{$value['idCol']};
6025         if ($key == 'groups') {
6026           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6027           continue;
6028         }
6029
6030         if (CRM_Utils_System::isNull($val)) {
6031           $dao->$key = NULL;
6032         }
6033         elseif (!empty($value['pseudoconstant'])) {
6034           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6035           // and we can use the metadata to figure it out.
6036           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6037           // more metadata based choices.
6038           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6039             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6040             foreach ($dbValues as $pseudoValue) {
6041               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6042             }
6043
6044             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6045             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
6046             if ($usedForAPI && $realFieldName) {
6047               // normally we would see 2 fields returned for pseudoConstants. An exception is
6048               // preferred_communication_method where there is no id-variant.
6049               // For the api we prioritise getting the real data returned.
6050               // over the resolved version
6051               $dao->$realFieldName = $dbValues;
6052             }
6053
6054           }
6055           else {
6056             // This is basically the same as the default but since we have the bao we can use
6057             // a cached function.
6058             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6059           }
6060         }
6061         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6062           //preserve id value
6063           $idColumn = "{$key}_id";
6064           $dao->$idColumn = $val;
6065
6066           if ($key == 'state_province_name') {
6067             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6068           }
6069           else {
6070             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6071           }
6072         }
6073         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6074                 // old bad code:
6075 //          $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6076          // iank: Return abbreviation if its a province from a country that uses abbreviations,
6077          // otherwise return the name of the province.
6078          // https://lab.civicrm.org/dev/core/issues/724
6079          if (is_numeric(CRM_Core_PseudoConstant::stateProvinceAbbreviation($val))) {
6080            $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6081          } else {
6082            $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6083          }
6084         }
6085         // @todo handle this in the section above for pseudoconstants.
6086         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6087           // @todo define bao on this & merge into the above condition.
6088           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6089
6090           if ($value['pseudoField'] == 'participant_role') {
6091             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6092             foreach ($viewValues as $k => $v) {
6093               $viewValues[$k] = $pseudoOptions[$v];
6094             }
6095           }
6096           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6097         }
6098         else {
6099           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6100           $dao->$key = $labels[$val];
6101         }
6102
6103         // return converted values in array format
6104         if ($return) {
6105           if (strpos($key, '-') !== FALSE) {
6106             $keyVal = explode('-', $key);
6107             $current = &$values;
6108             $lastElement = array_pop($keyVal);
6109             foreach ($keyVal as $v) {
6110               if (!array_key_exists($v, $current)) {
6111                 $current[$v] = [];
6112               }
6113               $current = &$current[$v];
6114             }
6115             $current[$lastElement] = $dao->$key;
6116           }
6117           else {
6118             $values[$key] = $dao->$key;
6119           }
6120         }
6121       }
6122     }
6123     if (!$usedForAPI) {
6124       foreach ($this->legacyHackedFields as $realField => $labelField) {
6125         // This is a temporary routine for handling these fields while
6126         // we figure out how to handled them based on metadata in
6127         /// export and search builder. CRM-19815, CRM-19830.
6128         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6129           $dao->$realField = $dao->$labelField;
6130         }
6131       }
6132     }
6133     return $values;
6134   }
6135
6136   /**
6137    * Include pseudo fields LEFT JOIN.
6138    * @param string|array $sort can be a object or string
6139    *
6140    * @return array|NULL
6141    */
6142   public function includePseudoFieldsJoin($sort) {
6143     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6144       return NULL;
6145     }
6146     $sort = is_string($sort) ? $sort : $sort->orderBy();
6147     $present = [];
6148
6149     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6150       if (!empty($value['table'])) {
6151         $regex = "/({$value['table']}\.|{$name})/";
6152         if (preg_match($regex, $sort)) {
6153           $this->_elemnt[$value['element']] = 1;
6154           $this->_select[$value['element']] = $value['select'];
6155           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6156           $present[$value['table']] = $value['join'];
6157         }
6158       }
6159     }
6160     $presentSimpleFrom = $present;
6161
6162     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6163       array_key_exists('civicrm_country', $presentSimpleFrom)
6164     ) {
6165       unset($presentSimpleFrom['civicrm_country']);
6166     }
6167     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6168       array_key_exists('civicrm_country', $present)
6169     ) {
6170       unset($present['civicrm_country']);
6171     }
6172
6173     $presentClause = $presentSimpleFromClause = NULL;
6174     if (!empty($present)) {
6175       $presentClause = implode(' ', $present);
6176     }
6177     if (!empty($presentSimpleFrom)) {
6178       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6179     }
6180
6181     $this->_fromClause = $this->_fromClause . $presentClause;
6182     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6183
6184     return [$presentClause, $presentSimpleFromClause];
6185   }
6186
6187   /**
6188    * Build qill for field.
6189    *
6190    * Qill refers to the query detail visible on the UI.
6191    *
6192    * @param string $daoName
6193    * @param string $fieldName
6194    * @param mixed $fieldValue
6195    * @param string $op
6196    * @param array $pseudoExtraParam
6197    * @param int $type
6198    *   Type of the field per CRM_Utils_Type
6199    *
6200    * @return array
6201    */
6202   public static function buildQillForFieldValue(
6203     $daoName,
6204     $fieldName,
6205     $fieldValue,
6206     $op,
6207     $pseudoExtraParam = [],
6208     $type = CRM_Utils_Type::T_STRING
6209   ) {
6210     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6211
6212     //API usually have fieldValue format as array(operator => array(values)),
6213     //so we need to separate operator out of fieldValue param
6214     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6215       $op = key($fieldValue);
6216       $fieldValue = $fieldValue[$op];
6217     }
6218
6219     // if Operator chosen is NULL/EMPTY then
6220     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6221       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6222     }
6223
6224     // @todo - if the right BAO is passed in special handling for the below
6225     // fields should not be required. testQillOptions.
6226     if ($fieldName == 'country_id') {
6227       $pseudoOptions = CRM_Core_PseudoConstant::country();
6228     }
6229     elseif ($fieldName == 'county_id') {
6230       $pseudoOptions = CRM_Core_PseudoConstant::county();
6231     }
6232     elseif ($fieldName == 'world_region') {
6233       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6234     }
6235     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6236       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6237       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6238     }
6239     elseif ($fieldName == 'contribution_product_id') {
6240       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6241     }
6242     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6243       $pseudoOptions = CRM_Core_PseudoConstant::group();
6244     }
6245     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6246       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6247     }
6248     elseif ($daoName) {
6249       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6250     }
6251
6252     if (is_array($fieldValue)) {
6253       $qillString = [];
6254       if (!empty($pseudoOptions)) {
6255         foreach ((array) $fieldValue as $val) {
6256           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6257         }
6258         $fieldValue = implode(', ', $qillString);
6259       }
6260       else {
6261         if ($type == CRM_Utils_Type::T_DATE) {
6262           foreach ($fieldValue as $index => $value) {
6263             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6264           }
6265         }
6266         $separator = ', ';
6267         // @todo - this is a bit specific (one operator).
6268         // However it is covered by a unit test so can be altered later with
6269         // some confidence.
6270         if ($op === 'BETWEEN') {
6271           $separator = ' AND ';
6272         }
6273         $fieldValue = implode($separator, $fieldValue);
6274       }
6275     }
6276     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6277       $fieldValue = $pseudoOptions[$fieldValue];
6278     }
6279     elseif ($type === CRM_Utils_Type::T_DATE) {
6280       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6281     }
6282
6283     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6284   }
6285
6286   /**
6287    * Get the qill (search description for field) for the specified field.
6288    *
6289    * @param string $daoName
6290    * @param string $name
6291    * @param string $value
6292    * @param string|array $op
6293    * @param string $label
6294    *
6295    * @return string
6296    */
6297   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6298     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6299     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6300   }
6301
6302   /**
6303    * Alter value to reflect wildcard settings.
6304    *
6305    * The form will have tried to guess whether this is a good field to wildcard but there is
6306    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6307    * or only the end of the string
6308    *
6309    * @param bool $wildcard
6310    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6311    * @param string $op
6312    *   Generally '=' or 'LIKE'.
6313    * @param string $value
6314    *   The search string.
6315    *
6316    * @return string
6317    */
6318   public static function getWildCardedValue($wildcard, $op, $value) {
6319     if ($wildcard && $op === 'LIKE') {
6320       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6321         $value = "%$value";
6322       }
6323       if (substr($value, -1, 1) != '%') {
6324         $value = "$value%";
6325       }
6326     }
6327     return "$value";
6328   }
6329
6330   /**
6331    * Process special fields of Search Form in OK (Operator in Key) format
6332    *
6333    * @param array $formValues
6334    * @param array $specialFields
6335    *    Special params to be processed
6336    * @param array $changeNames
6337    *   Array of fields whose name should be changed
6338    */
6339   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6340     // Array of special fields whose value are considered only for NULL or EMPTY operators
6341     $nullableFields = ['contribution_batch_id'];
6342
6343     foreach ($specialFields as $element) {
6344       $value = $formValues[$element] ?? NULL;
6345       if ($value) {
6346         if (is_array($value)) {
6347           if (array_key_exists($element, $changeNames)) {
6348             unset($formValues[$element]);
6349             $element = $changeNames[$element];
6350           }
6351           $formValues[$element] = ['IN' => $value];
6352         }
6353         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6354           $formValues[$element] = [$value => 1];
6355         }
6356         elseif (!in_array($element, $nullableFields)) {
6357           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6358           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6359           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6360         }
6361       }
6362     }
6363   }
6364
6365   /**
6366    * Parse and assimilate the various sort options.
6367    *
6368    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6369    * `email`), the related table may be joined automatically.
6370    *
6371    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6372    * it's been extracted to a standalone function.
6373    *
6374    * @param string|CRM_Utils_Sort $sort
6375    *   The order by string.
6376    * @param null $sortOrder
6377    *   Who knows? Hu knows. He who knows Hu knows who.
6378    *
6379    * @return string
6380    *   list(string $orderByClause, string $additionalFromClause).
6381    *
6382    * @throws \CRM_Core_Exception
6383    */
6384   protected function prepareOrderBy($sort, $sortOrder) {
6385     $orderByArray = [];
6386     $orderBy = '';
6387
6388     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6389       isset($this->_distinctComponentClause)
6390     ) {
6391       if ($sort) {
6392         if (is_string($sort)) {
6393           $orderBy = $sort;
6394         }
6395         else {
6396           $orderBy = trim($sort->orderBy());
6397         }
6398         // Deliberately remove the backticks again, as they mess up the evil
6399         // string munging below. This balanced by re-escaping before use.
6400         $orderBy = str_replace('`', '', $orderBy);
6401
6402         if (!empty($orderBy)) {
6403           // this is special case while searching for
6404           // change log CRM-1718
6405           if (preg_match('/sort_name/i', $orderBy)) {
6406             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6407           }
6408
6409           if ($sortOrder) {
6410             $orderBy .= " $sortOrder";
6411           }
6412
6413           // always add contact_a.id to the ORDER clause
6414           // so the order is deterministic
6415           if (strpos('contact_a.id', $orderBy) === FALSE) {
6416             $orderBy .= ", contact_a.id";
6417           }
6418         }
6419       }
6420       else {
6421         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6422       }
6423     }
6424     if (!$orderBy) {
6425       return NULL;
6426     }
6427     // Remove this here & add it at the end for simplicity.
6428     $order = trim($orderBy);
6429     $orderByArray = explode(',', $order);
6430
6431     foreach ($orderByArray as $orderByClause) {
6432       $orderByClauseParts = explode(' ', trim($orderByClause));
6433       $field = $orderByClauseParts[0];
6434       $direction = $orderByClauseParts[1] ?? 'asc';
6435       $fieldSpec = $this->getMetadataForRealField($field);
6436
6437       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6438       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6439       // add keys to $this->fields to represent the actual keys for locations.
6440       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6441         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6442         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6443       }
6444
6445       if ($this->_returnProperties === []) {
6446         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6447           $this->_tables[$fieldSpec['table_name']] = 1;
6448           $order = $fieldSpec['where'] . ' ' . $direction;
6449         }
6450
6451       }
6452       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6453       // add to cfIDs array if not present
6454       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6455         $this->_cfIDs[$cfID] = [];
6456         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6457         $this->_customQuery->query();
6458         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6459         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6460       }
6461
6462       // By replacing the join to the option value table with the mysql construct
6463       // ORDER BY field('contribution_status_id', 2,1,4)
6464       // we can remove a join. In the case of the option value join it is
6465       /// a join known to cause slow queries.
6466       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6467       // matching an id+name parrern in the
6468       // first instance for scope reasons. They require slightly different handling as the column (label)
6469       // is not declared for them.
6470       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6471       // escaping.
6472       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6473       if (!empty($pseudoConstantMetadata)
6474       ) {
6475         if (!empty($pseudoConstantMetadata['optionGroupName'])
6476           || $this->isPseudoFieldAnFK($fieldSpec)
6477         ) {
6478           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6479           if (empty($fieldSpec['bao'])) {
6480             continue;
6481           }
6482           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name']);
6483           natcasesort($sortedOptions);
6484           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6485           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6486           // this might give some early warning.
6487           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6488           // use where if it's set to fully qualify ambiguous column names
6489           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6490           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6491           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6492         }
6493         //CRM-12565 add "`" around $field if it is a pseudo constant
6494         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6495         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6496           $order = str_replace($field, "`{$field}`", $order);
6497         }
6498       }
6499     }
6500
6501     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6502     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6503
6504     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6505     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6506     if ($order) {
6507       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6508       return ' ORDER BY ' . $order;
6509     }
6510   }
6511
6512   /**
6513    * Convert a string of group IDs to a string of group labels.
6514    *
6515    * The original string may include duplicates and groups the user does not have
6516    * permission to see.
6517    *
6518    * @param CRM_Core_DAO $dao
6519    * @param string $val
6520    *
6521    * @return string
6522    */
6523   public function convertGroupIDStringToLabelString(&$dao, $val) {
6524     $groupIDs = explode(',', $val);
6525     // Note that groups that the user does not have permission to will be excluded (good).
6526     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6527     return implode(', ', $groups);
6528
6529   }
6530
6531   /**
6532    * Set the qill and where properties for a field.
6533    *
6534    * This function is intended as a short-term function to encourage refactoring
6535    * & re-use - but really we should just have less special-casing.
6536    *
6537    * @param string $name
6538    * @param string $op
6539    * @param string|array $value
6540    * @param string $grouping
6541    * @param array $field
6542    *
6543    * @throws \CRM_Core_Exception
6544    */
6545   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6546     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6547     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6548     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6549       1 => $field['title'],
6550       2 => $qillop,
6551       3 => $qillVal,
6552     ]);
6553   }
6554
6555   /**
6556    * Has the pseudoconstant of the field been requested.
6557    *
6558    * For example if the field is payment_instrument_id then it
6559    * has been requested if either payment_instrument_id or payment_instrument
6560    * have been requested. Payment_instrument is the option groun name field value.
6561    *
6562    * @param array $field
6563    * @param string $fieldName
6564    *   The unique name of the field - ie. the one it will be aliased to in the query.
6565    *
6566    * @return bool
6567    */
6568   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6569     $realField = $this->getMetadataForRealField($fieldName);
6570     if (!isset($realField['pseudoconstant'])) {
6571       return FALSE;
6572     }
6573     $pseudoConstant = $realField['pseudoconstant'];
6574     if (empty($pseudoConstant['optionGroupName']) &&
6575       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6576       // We are increasing our pseudoconstant handling - but still very cautiously,
6577       // hence the check for labelColumn === name
6578       return FALSE;
6579     }
6580
6581     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6582       return TRUE;
6583     }
6584     if (!empty($this->_returnProperties[$fieldName])) {
6585       return TRUE;
6586     }
6587     // Is this still required - the above goes off the unique name. Test with things like
6588     // communication_preferences & prefix_id.
6589     if (!empty($this->_returnProperties[$field['name']])) {
6590       return TRUE;
6591     }
6592     return FALSE;
6593   }
6594
6595   /**
6596    * Get Select Clause.
6597    *
6598    * @return string
6599    */
6600   public function getSelect() {
6601     $select = 'SELECT ';
6602     if (isset($this->_distinctComponentClause)) {
6603       $select .= "{$this->_distinctComponentClause}, ";
6604     }
6605     $select .= implode(', ', $this->_select);
6606     return $select;
6607   }
6608
6609   /**
6610    * Add basic statistics to the summary.
6611    *
6612    * @param array $summary
6613    * @param string $where
6614    * @param string $from
6615    *
6616    * @return array
6617    * @throws \CRM_Core_Exception
6618    */
6619   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6620     $summary['total']['count'] = 0;
6621     $summary['total']['amount'] = $summary['total']['avg'] = [];
6622
6623     $query = "
6624       SELECT COUNT( conts.total_amount ) as total_count,
6625         SUM(   conts.total_amount ) as total_amount,
6626         AVG(   conts.total_amount ) as total_avg,
6627         conts.currency              as currency
6628       FROM (
6629         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6630         civicrm_contribution.currency
6631         $from
6632         $where AND civicrm_contribution.contribution_status_id = 1
6633       GROUP BY civicrm_contribution.id
6634     ) as conts
6635     GROUP BY currency";
6636
6637     $dao = CRM_Core_DAO::executeQuery($query);
6638
6639     while ($dao->fetch()) {
6640       $summary['total']['count'] += $dao->total_count;
6641       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6642       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6643     }
6644
6645     if (!empty($summary['total']['amount'])) {
6646       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6647       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6648     }
6649     else {
6650       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6651     }
6652     return $summary;
6653   }
6654
6655   /**
6656    * Add basic soft credit statistics to summary array.
6657    *
6658    * @param array $summary
6659    * @param string $where
6660    * @param string $from
6661    *
6662    * @throws \CRM_Core_Exception
6663    */
6664   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6665     $query = "
6666       SELECT COUNT( conts.total_amount ) as total_count,
6667       SUM(   conts.total_amount ) as total_amount,
6668       AVG(   conts.total_amount ) as total_avg,
6669       conts.currency as currency
6670       FROM (
6671         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6672         $from
6673         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6674         GROUP BY civicrm_contribution_soft.id
6675       ) as conts
6676       GROUP BY currency";
6677
6678     $dao = CRM_Core_DAO::executeQuery($query);
6679     $summary['soft_credit']['count'] = 0;
6680     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6681     while ($dao->fetch()) {
6682       $summary['soft_credit']['count'] += $dao->total_count;
6683       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6684       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6685     }
6686     if (!empty($summary['soft_credit']['amount'])) {
6687       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6688       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6689     }
6690     else {
6691       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6692     }
6693   }
6694
6695   /**
6696    * Add basic stats about cancelled contributions to the summary.
6697    *
6698    * @param array $summary
6699    * @param string $where
6700    * @param string $from
6701    *
6702    * @throws \CRM_Core_Exception
6703    */
6704   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6705     $query = "
6706       SELECT COUNT( conts.total_amount ) as cancel_count,
6707        SUM(   conts.total_amount ) as cancel_amount,
6708        AVG(   conts.total_amount ) as cancel_avg,
6709        conts.currency              as currency
6710         FROM (
6711       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6712       $from
6713       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6714       GROUP BY civicrm_contribution.id
6715     ) as conts
6716     GROUP BY currency";
6717
6718     $dao = CRM_Core_DAO::executeQuery($query);
6719     $summary['cancel'] = ['count' => 0, 'amount' => 0, 'avg' => 0];
6720     if ($dao->N <= 1) {
6721       if ($dao->fetch()) {
6722         $summary['cancel']['count'] = $dao->cancel_count;
6723         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6724         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6725       }
6726     }
6727     else {
6728       $summary['cancel']['count'] = 0;
6729       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6730       while ($dao->fetch()) {
6731         $summary['cancel']['count'] += $dao->cancel_count;
6732         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6733         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6734       }
6735       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6736       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6737     }
6738   }
6739
6740   /**
6741    * Create the sql query for an contact search.
6742    *
6743    * @param int $offset
6744    *   The offset for the query.
6745    * @param int $rowCount
6746    *   The number of rows to return.
6747    * @param string|CRM_Utils_Sort $sort
6748    *   The order by string.
6749    * @param bool $count
6750    *   Is this a count only query ?.
6751    * @param bool $includeContactIds
6752    *   Should we include contact ids?.
6753    * @param bool $sortByChar
6754    *   If true returns the distinct array of first characters for search results.
6755    * @param bool $groupContacts
6756    *   If true, return only the contact ids.
6757    * @param string $additionalWhereClause
6758    *   If the caller wants to further restrict the search (used for components).
6759    * @param null $sortOrder
6760    * @param string $additionalFromClause
6761    *   Should be clause with proper joins, effective to reduce where clause load.
6762    *
6763    * @param bool $skipOrderAndLimit
6764    *
6765    * @return string
6766    *
6767    * @throws \CRM_Core_Exception
6768    */
6769   public function getSearchSQL(
6770     $offset = 0, $rowCount = 0, $sort = NULL,
6771     $count = FALSE, $includeContactIds = FALSE,
6772     $sortByChar = FALSE, $groupContacts = FALSE,
6773     $additionalWhereClause = NULL, $sortOrder = NULL,
6774     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6775
6776     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6777
6778     if ($sortByChar) {
6779       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6780       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6781     }
6782
6783     if ($skipOrderAndLimit) {
6784       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6785       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6786     }
6787     else {
6788       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6789     }
6790     return $query;
6791   }
6792
6793   /**
6794    * Get the component parts of the search query as an array.
6795    *
6796    * @param int $offset
6797    *   The offset for the query.
6798    * @param int $rowCount
6799    *   The number of rows to return.
6800    * @param string|CRM_Utils_Sort $sort
6801    *   The order by string.
6802    * @param bool $count
6803    *   Is this a count only query ?.
6804    * @param bool $includeContactIds
6805    *   Should we include contact ids?.
6806    * @param bool $sortByChar
6807    *   If true returns the distinct array of first characters for search results.
6808    * @param bool $groupContacts
6809    *   If true, return only the contact ids.
6810    * @param string $additionalWhereClause
6811    *   If the caller wants to further restrict the search (used for components).
6812    * @param null $sortOrder
6813    * @param string $additionalFromClause
6814    *   Should be clause with proper joins, effective to reduce where clause load.
6815    *
6816    * @return array
6817    * @throws \CRM_Core_Exception
6818    */
6819   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6820     $count = FALSE, $includeContactIds = FALSE,
6821     $sortByChar = FALSE, $groupContacts = FALSE,
6822     $additionalWhereClause = NULL, $sortOrder = NULL,
6823     $additionalFromClause = NULL) {
6824     if ($includeContactIds) {
6825       $this->_includeContactIds = TRUE;
6826       $this->_whereClause = $this->whereClause();
6827     }
6828     $onlyDeleted = in_array([
6829       'deleted_contacts',
6830       '=',
6831       '1',
6832       '0',
6833       '0',
6834     ], $this->_params);
6835
6836     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6837     // and that contact happens to be deleted, set $onlyDeleted to true
6838     foreach ($this->_params as $values) {
6839       $name = $values[0] ?? NULL;
6840       $op = $values[1] ?? NULL;
6841       $value = $values[2] ?? NULL;
6842       if ($name === 'contact_id' and $op === '=') {
6843         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6844           $onlyDeleted = TRUE;
6845         }
6846         break;
6847       }
6848     }
6849
6850     // building the query string
6851     $groupBy = $groupByCols = NULL;
6852     if (!$count) {
6853       if (isset($this->_groupByComponentClause)) {
6854         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6855         $groupByCols = explode(', ', $groupByCols);
6856       }
6857       elseif ($this->_useGroupBy) {
6858         $groupByCols = ['contact_a.id'];
6859       }
6860     }
6861     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6862       $groupByCols = ['civicrm_activity.id'];
6863     }
6864     if (!empty($groupByCols)) {
6865       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6866     }
6867
6868     $order = $orderBy = '';
6869     if (!$count) {
6870       if (!$sortByChar) {
6871         $order = $this->prepareOrderBy($sort, $sortOrder);
6872       }
6873     }
6874     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6875     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6876     //     clause. This will impact the search query output.
6877     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6878
6879     if ($disableFullGroupByMode) {
6880       CRM_Core_DAO::disableFullGroupByMode();
6881     }
6882
6883     // CRM-15231
6884     $this->_sort = $sort;
6885
6886     //CRM-15967
6887     $this->includePseudoFieldsJoin($sort);
6888
6889     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6890
6891     if ($additionalWhereClause) {
6892       $where = $where . ' AND ' . $additionalWhereClause;
6893     }
6894
6895     //additional from clause should be w/ proper joins.
6896     if ($additionalFromClause) {
6897       $from .= "\n" . $additionalFromClause;
6898     }
6899
6900     // if we are doing a transform, do it here
6901     // use the $from, $where and $having to get the contact ID
6902     if ($this->_displayRelationshipType) {
6903       $this->filterRelatedContacts($from, $where, $having);
6904     }
6905     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6906
6907     return [
6908       'select' => $select,
6909       'from' => $from,
6910       'where' => $where,
6911       'order_by' => $order,
6912       'group_by' => $groupBy,
6913       'having' => $having,
6914       'limit' => $limit,
6915     ];
6916   }
6917
6918   /**
6919    * Get the metadata for a given field.
6920    *
6921    * @param string $fieldName
6922    *
6923    * @return array
6924    */
6925   protected function getMetadataForField($fieldName) {
6926     if ($fieldName === 'contact_a.id') {
6927       // This seems to be the only anomaly.
6928       $fieldName = 'id';
6929     }
6930     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6931     $field = $this->_fields[$fieldName] ?? $pseudoField;
6932     $field = array_merge($field, $pseudoField);
6933     if (!empty($field) && empty($field['name'])) {
6934       // standardising field formatting here - over time we can phase out variants.
6935       // all paths using this currently unit tested
6936       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6937     }
6938     return $field;
6939   }
6940
6941   /**
6942    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6943    *
6944    * @param string $fieldName
6945    *
6946    * @return array
6947    */
6948   public function getMetadataForRealField($fieldName) {
6949     $field = $this->getMetadataForField($fieldName);
6950     if (!empty($field['is_pseudofield_for'])) {
6951       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6952       $field['pseudofield_name'] = $fieldName;
6953     }
6954     elseif (!empty($field['pseudoconstant'])) {
6955       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6956         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6957         if (empty($field['table_name'])) {
6958           if (!empty($field['where'])) {
6959             $field['table_name'] = explode('.', $field['where'])[0];
6960           }
6961           else {
6962             $field['table_name'] = 'civicrm_contact';
6963           }
6964         }
6965       }
6966     }
6967     return $field;
6968   }
6969
6970   /**
6971    * Get the field datatype, using the type in the database rather than the pseudofield, if a pseudofield.
6972    *
6973    * @param string $fieldName
6974    *
6975    * @return string
6976    */
6977   public function getDataTypeForRealField($fieldName) {
6978     return CRM_Utils_Type::typeToString($this->getMetadataForRealField($fieldName)['type']);
6979   }
6980
6981   /**
6982    * If we have a field that is better rendered via the pseudoconstant handled them here.
6983    *
6984    * Rather than joining in the additional table we render the option value on output.
6985    *
6986    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6987    * carefully with test coverage but aim to extend.
6988    *
6989    * @param string $name
6990    */
6991   protected function addPseudoconstantFieldToSelect($name) {
6992     $field = $this->getMetadataForRealField($name);
6993     $realFieldName = $field['name'];
6994     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6995     if ($pseudoFieldName) {
6996       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6997       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6998         'pseudoField' => $pseudoFieldName,
6999         'idCol' => $realFieldName,
7000         'field_name' => $field['name'],
7001         'bao' => $field['bao'],
7002         'pseudoconstant' => $field['pseudoconstant'],
7003       ];
7004     }
7005
7006     $this->_tables[$field['table_name']] = 1;
7007     $this->_element[$realFieldName] = 1;
7008     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
7009   }
7010
7011   /**
7012    * Is this pseudofield a foreign key constraint.
7013    *
7014    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
7015    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
7016    * array which is done for contributions which are mostly handled as pseudoconstants.
7017    *
7018    * @param $fieldSpec
7019    *
7020    * @return bool
7021    */
7022   protected function isPseudoFieldAnFK($fieldSpec) {
7023     if (empty($fieldSpec['FKClassName'])
7024       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
7025       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
7026       return FALSE;
7027     }
7028     return TRUE;
7029   }
7030
7031   /**
7032    * Is the field a relative date field.
7033    *
7034    * @param string $fieldName
7035    *
7036    * @return bool
7037    */
7038   protected function isARelativeDateField($fieldName) {
7039     if (substr($fieldName, -9, 9) !== '_relative') {
7040       return FALSE;
7041     }
7042     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
7043     return isset($this->_fields[$realField]);
7044   }
7045
7046   /**
7047    * Get the specifications for the field, if available.
7048    *
7049    * @param string $fieldName
7050    *   Fieldname as displayed on the form.
7051    *
7052    * @return array
7053    */
7054   public function getFieldSpec($fieldName) {
7055     if (isset($this->_fields[$fieldName])) {
7056       $fieldSpec = $this->_fields[$fieldName];
7057       if (!empty($fieldSpec['is_pseudofield_for'])) {
7058         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
7059       }
7060       return $fieldSpec;
7061     }
7062     $lowFieldName = str_replace('_low', '', $fieldName);
7063     if (isset($this->_fields[$lowFieldName])) {
7064       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
7065     }
7066     $highFieldName = str_replace('_high', '', $fieldName);
7067     if (isset($this->_fields[$highFieldName])) {
7068       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
7069     }
7070     return [];
7071   }
7072
7073   public function buildWhereForDate() {
7074
7075   }
7076
7077   /**
7078    * Is the field a relative date field.
7079    *
7080    * @param string $fieldName
7081    *
7082    * @return bool
7083    */
7084   protected function isADateRangeField($fieldName) {
7085     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7086       return FALSE;
7087     }
7088     return !empty($this->getFieldSpec($fieldName));
7089   }
7090
7091   /**
7092    * @param $values
7093    */
7094   protected function buildRelativeDateQuery(&$values) {
7095     $value = $values[2] ?? NULL;
7096     if (empty($value)) {
7097       return;
7098     }
7099     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7100     $fieldSpec = $this->_fields[$fieldName];
7101     $tableName = $fieldSpec['table_name'];
7102     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7103     $grouping = $values[3] ?? NULL;
7104     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7105     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7106     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7107
7108     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7109     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7110     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7111
7112     $where = $fieldSpec['where'];
7113     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7114       // Special handling for contact table as it has a known alias in advanced search.
7115       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7116       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7117     }
7118
7119     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7120     if ($fieldName === 'relation_active_period_date') {
7121       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7122       // Not sure the  'right' fix.
7123       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7124       return;
7125     }
7126
7127     if (empty($dates[0])) {
7128       // ie. no start date we only have end date
7129       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7130     }
7131     elseif (empty($dates[1])) {
7132
7133       // ie. no end date we only have start date
7134       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7135     }
7136     else {
7137       // we have start and end dates.
7138       if ($secondWhere !== $where) {
7139         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7140       }
7141       else {
7142         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7143       }
7144     }
7145   }
7146
7147   /**
7148    * Build the query for a date field if it is a _high or _low field.
7149    *
7150    * @param $values
7151    *
7152    * @return bool
7153    * @throws \CRM_Core_Exception
7154    */
7155   public function buildDateRangeQuery($values) {
7156     if ($this->isADateRangeField($values[0])) {
7157       $fieldSpec = $this->getFieldSpec($values[0]);
7158       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7159       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7160       return TRUE;
7161     }
7162     return FALSE;
7163   }
7164
7165   /**
7166    * Add the address table into the query.
7167    *
7168    * @param string $tableKey
7169    * @param string $joinCondition
7170    *
7171    * @return array
7172    *   - alias name
7173    *   - address join.
7174    */
7175   protected function addAddressTable($tableKey, $joinCondition) {
7176     $tName = "$tableKey-address";
7177     $aName = "`$tableKey-address`";
7178     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7179     $this->_element["{$tName}_id"] = 1;
7180     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7181     $this->_tables[$tName] = $addressJoin;
7182
7183     return [
7184       $aName,
7185       $addressJoin,
7186     ];
7187   }
7188
7189   /**
7190    * Get the clause for group status.
7191    *
7192    * @param int $grouping
7193    *
7194    * @return string
7195    */
7196   protected function getGroupStatusClause($grouping) {
7197     $statuses = $this->getSelectedGroupStatuses($grouping);
7198     return "status IN (" . implode(', ', $statuses) . ")";
7199   }
7200
7201   /**
7202    * Get an array of the statuses that have been selected.
7203    *
7204    * @param string $grouping
7205    *
7206    * @return array
7207    *
7208    * @throws \CRM_Core_Exception
7209    */
7210   protected function getSelectedGroupStatuses($grouping) {
7211     $statuses = [];
7212     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7213     if ($gcsValues &&
7214       is_array($gcsValues[2])
7215     ) {
7216       foreach ($gcsValues[2] as $k => $v) {
7217         if ($v) {
7218           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7219         }
7220       }
7221     }
7222     else {
7223       $statuses[] = "'Added'";
7224     }
7225     return $statuses;
7226   }
7227
7228   /**
7229    * Get the qill value for the field.
7230    *
7231    * @param string $name
7232    * @param array|int|string $value
7233    * @param string $op
7234    * @param array $fieldSpec
7235    * @param string $labelOverride
7236    *   Label override, if required.
7237    *
7238    * @return string
7239    */
7240   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7241     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7242     return (string) ts("%1 %2 %3", [
7243       1 => $labelOverride ?? $fieldSpec['title'],
7244       2 => $qillop,
7245       3 => $qillVal,
7246     ]);
7247   }
7248
7249   /**
7250    * Where handling for any field with adequately defined metadata.
7251    *
7252    * @param array $fieldSpec
7253    * @param string $name
7254    * @param string|array|int $value
7255    * @param string $op
7256    * @param string|int $grouping
7257    *
7258    * @throws \CRM_Core_Exception
7259    */
7260   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7261     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7262     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7263     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7264       $this->_tables[$fieldSpec['table_name']] = 1;
7265     }
7266     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7267       $this->_whereTables[$fieldSpec['table_name']] = 1;
7268     }
7269   }
7270
7271   /**
7272    * Get the qill for the relative date range.
7273    *
7274    * @param string|null $from
7275    * @param string|null $to
7276    * @param string $fieldTitle
7277    * @param string $relativeRange
7278    *
7279    * @return string
7280    */
7281   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7282     if (!$from) {
7283       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7284     }
7285     if (!$to) {
7286       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7287     }
7288     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7289       CRM_Utils_Date::customFormat($from),
7290       CRM_Utils_Date::customFormat($to),
7291     ]) . ')';
7292   }
7293
7294 }