CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * Are we in search mode.
 218    *
 219    * @var bool
 220    */
 221   public $_search = TRUE;
 222
 223   /**
 224    * Should we skip permission checking.
 225    *
 226    * @var bool
 227    */
 228   public $_skipPermission = FALSE;
 229
 230   /**
 231    * Should we skip adding of delete clause.
 232    *
 233    * @var bool
 234    */
 235   public $_skipDeleteClause = FALSE;
 236
 237   /**
 238    * Are we in strict mode (use equality over LIKE)
 239    *
 240    * @var bool
 241    */
 242   public $_strict = FALSE;
 243
 244   /**
 245    * What operator to use to group the clauses.
 246    *
 247    * @var string
 248    */
 249   public $_operator = 'AND';
 250
 251   public $_mode = 1;
 252
 253   /**
 254    * Should we only search on primary location.
 255    *
 256    * @var bool
 257    */
 258   public $_primaryLocation = TRUE;
 259
 260   /**
 261    * Are contact ids part of the query.
 262    *
 263    * @var bool
 264    */
 265   public $_includeContactIds = FALSE;
 266
 267   /**
 268    * Should we use the smart group cache.
 269    *
 270    * @var bool
 271    */
 272   public $_smartGroupCache = TRUE;
 273
 274   /**
 275    * Should we display contacts with a specific relationship type.
 276    *
 277    * @var string
 278    */
 279   public $_displayRelationshipType;
 280
 281   /**
 282    * Reference to the query object for custom values.
 283    *
 284    * @var Object
 285    */
 286   public $_customQuery;
 287
 288   /**
 289    * Should we enable the distinct clause, used if we are including
 290    * more than one group
 291    *
 292    * @var bool
 293    */
 294   public $_useDistinct = FALSE;
 295
 296   /**
 297    * Should we just display one contact record
 298    * @var bool
 299    */
 300   public $_useGroupBy = FALSE;
 301
 302   /**
 303    * The relationship type direction
 304    *
 305    * @var array
 306    */
 307   public static $_relType;
 308
 309   /**
 310    * The activity role
 311    *
 312    * @var array
 313    */
 314   public static $_activityRole;
 315
 316   /**
 317    * Consider the component activity type
 318    * during activity search.
 319    *
 320    * @var array
 321    */
 322   public static $_considerCompActivities;
 323
 324   /**
 325    * Consider with contact activities only,
 326    * during activity search.
 327    *
 328    * @var array
 329    */
 330   public static $_withContactActivitiesOnly;
 331
 332   /**
 333    * Use distinct component clause for component searches
 334    *
 335    * @var string
 336    */
 337   public $_distinctComponentClause;
 338
 339   public $_rowCountClause;
 340
 341   /**
 342    * Use groupBy component clause for component searches
 343    *
 344    * @var string
 345    */
 346   public $_groupByComponentClause;
 347
 348   /**
 349    * Track open panes, useful in advance search
 350    *
 351    * @var array
 352    */
 353   public static $_openedPanes = [];
 354
 355   /**
 356    * For search builder - which custom fields are location-dependent
 357    * @var array
 358    */
 359   public $_locationSpecificCustomFields = [];
 360
 361   /**
 362    * The tables which have a dependency on location and/or address
 363    *
 364    * @var array
 365    */
 366   public static $_dependencies = [
 367     'civicrm_state_province' => 1,
 368     'civicrm_country' => 1,
 369     'civicrm_county' => 1,
 370     'civicrm_address' => 1,
 371     'civicrm_location_type' => 1,
 372   ];
 373
 374   /**
 375    * List of location specific fields.
 376    * @var array
 377    */
 378   public static $_locationSpecificFields = [
 379     'street_address',
 380     'street_number',
 381     'street_name',
 382     'street_unit',
 383     'supplemental_address_1',
 384     'supplemental_address_2',
 385     'supplemental_address_3',
 386     'city',
 387     'postal_code',
 388     'postal_code_suffix',
 389     'geo_code_1',
 390     'geo_code_2',
 391     'state_province',
 392     'country',
 393     'county',
 394     'phone',
 395     'email',
 396     'im',
 397     'address_name',
 398     'master_id',
 399   ];
 400
 401   /**
 402    * Remember if we handle either end of a number or date range
 403    * so we can skip the other
 404    * @var array
 405    */
 406   protected $_rangeCache = [];
 407   /**
 408    * Set to true when $this->relationship is run to avoid adding twice.
 409    *
 410    * @var bool
 411    */
 412   protected $_relationshipValuesAdded = FALSE;
 413
 414   /**
 415    * Set to the name of the temp table if one has been created.
 416    *
 417    * @var string
 418    */
 419   public static $_relationshipTempTable;
 420
 421   public $_pseudoConstantsSelect = [];
 422
 423   public $_groupUniqueKey;
 424   public $_groupKeys = [];
 425
 426   /**
 427    * Class constructor which also does all the work.
 428    *
 429    * @param array $params
 430    * @param array $returnProperties
 431    * @param array $fields
 432    * @param bool $includeContactIds
 433    * @param bool $strict
 434    * @param bool|int $mode - mode the search is operating on
 435    *
 436    * @param bool $skipPermission
 437    * @param bool $searchDescendentGroups
 438    * @param bool $smartGroupCache
 439    * @param null $displayRelationshipType
 440    * @param string $operator
 441    * @param string $apiEntity
 442    * @param bool|null $primaryLocationOnly
 443    *
 444    * @throws \CRM_Core_Exception
 445    */
 446   public function __construct(
 447     $params = NULL, $returnProperties = NULL, $fields = NULL,
 448     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 449     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 450     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 451     $operator = 'AND',
 452     $apiEntity = NULL,
 453     $primaryLocationOnly = NULL
 454   ) {
 455     if ($primaryLocationOnly === NULL) {
 456       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 457     }
 458     $this->_primaryLocation = $primaryLocationOnly;
 459     $this->_params = &$params;
 460     if ($this->_params == NULL) {
 461       $this->_params = [];
 462     }
 463
 464     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 465       $this->_returnProperties = [];
 466     }
 467     elseif (empty($returnProperties)) {
 468       $this->_returnProperties = self::defaultReturnProperties($mode);
 469     }
 470     else {
 471       $this->_returnProperties = &$returnProperties;
 472     }
 473
 474     $this->_includeContactIds = $includeContactIds;
 475     $this->_strict = $strict;
 476     $this->_mode = $mode;
 477     $this->_skipPermission = $skipPermission;
 478     $this->_smartGroupCache = $smartGroupCache;
 479     $this->_displayRelationshipType = $displayRelationshipType;
 480     $this->setOperator($operator);
 481
 482     if ($fields) {
 483       $this->_fields = &$fields;
 484       $this->_search = FALSE;
 485       $this->_skipPermission = TRUE;
 486     }
 487     else {
 488       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 489       //  The legacy hacked fields will output as a string rather than their underlying type.
 490       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 491         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 492       }
 493       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 494       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 495       // No good option other than hard-coding metadata for this 'special' field in.
 496       $relationFields['relation_active_period_date'] = [
 497         'name' => 'relation_active_period_date',
 498         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 499         'title' => ts('Active Period'),
 500         'table_name' => 'civicrm_relationship',
 501         'where' => 'civicrm_relationship.start_date',
 502         'where_end' => 'civicrm_relationship.end_date',
 503         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 504       ];
 505       $this->_fields = array_merge($relationFields, $this->_fields);
 506
 507       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 508       unset($fields['note']);
 509       $this->_fields = array_merge($this->_fields, $fields);
 510
 511       // add activity fields
 512       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 513       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields('Case'));
 514       // Add hack as no unique name is defined for the field but the search form is in denial.
 515       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 516
 517       // add any fields provided by hook implementers
 518       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 519       $this->_fields = array_merge($this->_fields, $extFields);
 520     }
 521
 522     // basically do all the work once, and then reuse it
 523     $this->initialize($apiEntity);
 524   }
 525
 526   /**
 527    * Function which actually does all the work for the constructor.
 528    *
 529    * @param string $apiEntity
 530    *   The api entity being called.
 531    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 532    *
 533    * @throws \CRM_Core_Exception
 534    */
 535   public function initialize($apiEntity = NULL) {
 536     $this->_select = [];
 537     $this->_element = [];
 538     $this->_tables = [];
 539     $this->_whereTables = [];
 540     $this->_where = [];
 541     $this->_qill = [];
 542     $this->_cfIDs = [];
 543     $this->_paramLookup = [];
 544     $this->_having = [];
 545
 546     $this->_customQuery = NULL;
 547
 548     // reset cached static variables - CRM-5803
 549     self::$_activityRole = NULL;
 550     self::$_considerCompActivities = NULL;
 551     self::$_withContactActivitiesOnly = NULL;
 552
 553     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 554     $this->_element['contact_id'] = 1;
 555     $this->_tables['civicrm_contact'] = 1;
 556
 557     if (!empty($this->_params)) {
 558       $this->buildParamsLookup();
 559     }
 560
 561     $this->_whereTables = $this->_tables;
 562
 563     $this->selectClause($apiEntity);
 564     if (!empty($this->_cfIDs)) {
 565       // @todo This function is the select function but instead of running 'select' it
 566       // is running the whole query.
 567       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 568       $this->_customQuery->query();
 569       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 570       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 571       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 572     }
 573     $isForcePrimaryOnly = !empty($apiEntity);
 574     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 575     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 576       $component = 'contribution';
 577     }
 578     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 579       $component = 'membership';
 580     }
 581     if (isset($component) && !$this->_skipPermission) {
 582       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 583       CRM_Financial_BAO_FinancialType::buildPermissionedClause($this->_whereClause, $component);
 584     }
 585
 586     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 587     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 588
 589     $this->openedSearchPanes(TRUE);
 590   }
 591
 592   /**
 593    * Function for same purpose as convertFormValues.
 594    *
 595    * Like convert form values this function exists to pre-Process parameters from the form.
 596    *
 597    * It is unclear why they are different functions & likely relates to advances search
 598    * versus search builder.
 599    *
 600    * The direction we are going is having the form convert values to a standardised format &
 601    * moving away from weird & wonderful where clause switches.
 602    *
 603    * Fix and handle contact deletion nicely.
 604    *
 605    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 606    *
 607    * @see https://issues.civicrm.org/jira/browse/CRM-11971
 608    */
 609   public function buildParamsLookup() {
 610     $trashParamExists = FALSE;
 611     $paramByGroup = [];
 612     foreach ($this->_params as $k => $param) {
 613       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 614         $trashParamExists = TRUE;
 615       }
 616       if (!empty($param[3])) {
 617         $paramByGroup[$param[3]][$k] = $param;
 618       }
 619     }
 620
 621     if ($trashParamExists) {
 622       $this->_skipDeleteClause = TRUE;
 623
 624       //cycle through group sets and explicitly add trash param if not set
 625       foreach ($paramByGroup as $setID => $set) {
 626         if (
 627           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 628           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 629         ) {
 630           $this->_params[] = [
 631             'contact_is_deleted',
 632             '=',
 633             '0',
 634             $setID,
 635             '0',
 636           ];
 637         }
 638       }
 639     }
 640
 641     foreach ($this->_params as $value) {
 642       if (empty($value[0])) {
 643         continue;
 644       }
 645       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 646       if ($cfID) {
 647         if (!array_key_exists($cfID, $this->_cfIDs)) {
 648           $this->_cfIDs[$cfID] = [];
 649         }
 650         // Set wildcard value based on "and/or" selection
 651         foreach ($this->_params as $key => $param) {
 652           if ($param[0] == $value[0] . '_operator') {
 653             $value[4] = $param[2] == 'or';
 654             break;
 655           }
 656         }
 657         $this->_cfIDs[$cfID][] = $value;
 658       }
 659
 660       if (!array_key_exists($value[0], $this->_paramLookup)) {
 661         $this->_paramLookup[$value[0]] = [];
 662       }
 663       if ($value[0] !== 'group') {
 664         // Just trying to unravel how group interacts here! This whole function is weird.
 665         $this->_paramLookup[$value[0]][] = $value;
 666       }
 667     }
 668   }
 669
 670   /**
 671    * Some composite fields do not appear in the fields array hack to make them part of the query.
 672    *
 673    * @param $apiEntity
 674    *   The api entity being called.
 675    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 676    */
 677   public function addSpecialFields($apiEntity) {
 678     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 679     // if get called via Contact.get API having address_id as return parameter
 680     if ($apiEntity === 'Contact') {
 681       $special[] = 'address_id';
 682     }
 683     foreach ($special as $name) {
 684       if (!empty($this->_returnProperties[$name])) {
 685         if ($name === 'address_id') {
 686           $this->_tables['civicrm_address'] = 1;
 687           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 688           $this->_element['address_id'] = 1;
 689         }
 690         else {
 691           $this->_select[$name] = "contact_a.{$name} as $name";
 692           $this->_element[$name] = 1;
 693         }
 694       }
 695     }
 696   }
 697
 698   /**
 699    * Given a list of conditions in params and a list of desired
 700    * return Properties generate the required select and from
 701    * clauses. Note that since the where clause introduces new
 702    * tables, the initial attempt also retrieves all variables used
 703    * in the params list
 704    *
 705    * @param string $apiEntity
 706    *   The api entity being called.
 707    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 708    */
 709   public function selectClause($apiEntity = NULL) {
 710
 711     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 712     // param ($apiEntity) instead - presumably an oversight & 2 because
 713     // contact is not implemented as a component.
 714     $this->addSpecialFields($apiEntity);
 715
 716     foreach ($this->_fields as $name => $field) {
 717       // skip component fields
 718       // there are done by the alter query below
 719       // and need not be done on every field
 720       // @todo remove these & handle using metadata - only obscure fields
 721       // that are hack-added should need to be excluded from the main loop.
 722       if (
 723         (substr($name, 0, 12) === 'participant_') ||
 724         (substr($name, 0, 7) === 'pledge_') ||
 725         (substr($name, 0, 5) === 'case_')
 726       ) {
 727         continue;
 728       }
 729
 730       // redirect to activity select clause
 731       if (
 732         (substr($name, 0, 9) === 'activity_') ||
 733         ($name === 'parent_id')
 734       ) {
 735         CRM_Activity_BAO_Query::select($this);
 736       }
 737
 738       // if this is a hierarchical name, we ignore it
 739       $names = explode('-', $name);
 740       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 741         continue;
 742       }
 743
 744       // make an exception for special cases, to add the field in select clause
 745       $makeException = FALSE;
 746
 747       //special handling for groups/tags
 748       if (in_array($name, ['groups', 'tags', 'notes'])
 749         && isset($this->_returnProperties[substr($name, 0, -1)])
 750       ) {
 751         // @todo instead of setting make exception to get us into
 752         // an if clause that has handling for these fields buried with in it
 753         // move the handling to here.
 754         $makeException = TRUE;
 755       }
 756
 757       // since note has 3 different options we need special handling
 758       // note / note_subject / note_body
 759       if ($name === 'notes') {
 760         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 761           if (isset($this->_returnProperties[$noteField])) {
 762             $makeException = TRUE;
 763             break;
 764           }
 765         }
 766       }
 767
 768       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 769       if (
 770         !empty($this->_paramLookup[$name])
 771         || !empty($this->_returnProperties[$name])
 772         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 773         || $makeException
 774       ) {
 775         if ($cfID) {
 776           // add to cfIDs array if not present
 777           if (!array_key_exists($cfID, $this->_cfIDs)) {
 778             $this->_cfIDs[$cfID] = [];
 779           }
 780         }
 781         elseif (isset($field['where'])) {
 782           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 783           if (isset($tableName)) {
 784             if (!empty(self::$_dependencies[$tableName])) {
 785               $this->_tables['civicrm_address'] = 1;
 786               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 787               $this->_element['address_id'] = 1;
 788             }
 789
 790             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 791               $tableName === 'postal_greeting' || $tableName === 'addressee'
 792             ) {
 793               if ($tableName === 'im_provider') {
 794                 CRM_Core_OptionValue::select($this);
 795               }
 796
 797               if (in_array($tableName,
 798                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 799                 $this->_element["{$name}_id"] = 1;
 800                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 801                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 802                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 803                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 804
 805                 if ($tableName === 'email_greeting') {
 806                   // @todo bad join.
 807                   $this->_pseudoConstantsSelect[$name]['join']
 808                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 809                   $this->_pseudoConstantsSelect[$name]['join'] .=
 810                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 811                 }
 812                 elseif ($tableName === 'postal_greeting') {
 813                   // @todo bad join.
 814                   $this->_pseudoConstantsSelect[$name]['join']
 815                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 816                   $this->_pseudoConstantsSelect[$name]['join'] .=
 817                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 818                 }
 819                 elseif ($tableName == 'addressee') {
 820                   // @todo bad join.
 821                   $this->_pseudoConstantsSelect[$name]['join']
 822                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 823                   $this->_pseudoConstantsSelect[$name]['join'] .=
 824                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 825                 }
 826                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 827
 828                 //get display
 829                 $greetField = "{$name}_display";
 830                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 831                 $this->_element[$greetField] = 1;
 832                 //get custom
 833                 $greetField = "{$name}_custom";
 834                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 835                 $this->_element[$greetField] = 1;
 836               }
 837             }
 838             else {
 839               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 840                 $this->_tables[$tableName] = 1;
 841               }
 842
 843               // also get the id of the tableName
 844               $tName = substr($tableName, 8);
 845               if (in_array($tName, ['country', 'state_province', 'county'])) {
 846                 if ($tName == 'state_province') {
 847                   $this->_pseudoConstantsSelect['state_province_name'] = [
 848                     'pseudoField' => "{$tName}",
 849                     'idCol' => "{$tName}_id",
 850                     'bao' => 'CRM_Core_BAO_Address',
 851                     'table' => "civicrm_{$tName}",
 852                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 853                   ];
 854
 855                   $this->_pseudoConstantsSelect[$tName] = [
 856                     'pseudoField' => 'state_province_abbreviation',
 857                     'idCol' => "{$tName}_id",
 858                     'table' => "civicrm_{$tName}",
 859                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 860                   ];
 861                 }
 862                 else {
 863                   $this->_pseudoConstantsSelect[$name] = [
 864                     'pseudoField' => "{$tName}_id",
 865                     'idCol' => "{$tName}_id",
 866                     'bao' => 'CRM_Core_BAO_Address',
 867                     'table' => "civicrm_{$tName}",
 868                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 869                   ];
 870                 }
 871
 872                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 873                 $this->_element["{$tName}_id"] = 1;
 874               }
 875               elseif ($tName != 'contact') {
 876                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 877                 $this->_element["{$tName}_id"] = 1;
 878               }
 879
 880               //special case for phone
 881               if ($name == 'phone') {
 882                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 883                 $this->_element['phone_type_id'] = 1;
 884               }
 885
 886               // if IM then select provider_id also
 887               // to get "IM Service Provider" in a file to be exported, CRM-3140
 888               if ($name == 'im') {
 889                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 890                 $this->_element['provider_id'] = 1;
 891               }
 892
 893               if ($tName == 'contact' && $fieldName == 'organization_name') {
 894                 // special case, when current employer is set for Individual contact
 895                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 896               }
 897               elseif ($tName == 'contact' && $fieldName === 'id') {
 898                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 899               }
 900               elseif (in_array($tName, ['country', 'county'])) {
 901                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 902                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 903               }
 904               elseif ($tName == 'state_province') {
 905                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 906                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 907               }
 908               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 909                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 910                   $this->_select[$name] = "{$field['where']} as `$name`";
 911                 }
 912               }
 913               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 914                 $this->addPseudoconstantFieldToSelect($name);
 915               }
 916               else {
 917                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 918               }
 919               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 920                 $this->_element[$name] = 1;
 921               }
 922             }
 923           }
 924         }
 925         elseif ($name === 'tags') {
 926           //@todo move this handling outside the big IF & ditch $makeException
 927           $this->_useGroupBy = TRUE;
 928           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 929           $this->_element[$name] = 1;
 930           $this->_tables['civicrm_tag'] = 1;
 931           $this->_tables['civicrm_entity_tag'] = 1;
 932         }
 933         elseif ($name === 'groups') {
 934           //@todo move this handling outside the big IF & ditch $makeException
 935           $this->_useGroupBy = TRUE;
 936           // Duplicates will be created here but better to sort them out in php land.
 937           $this->_select[$name] = "
 938             CONCAT_WS(',',
 939             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 940             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 941           )
 942           as `groups`";
 943           $this->_element[$name] = 1;
 944           $this->_tables['civicrm_group_contact'] = 1;
 945           $this->_tables['civicrm_group_contact_cache'] = 1;
 946           $this->_pseudoConstantsSelect["{$name}"] = [
 947             'pseudoField' => "groups",
 948             'idCol' => 'groups',
 949           ];
 950         }
 951         elseif ($name === 'notes') {
 952           //@todo move this handling outside the big IF & ditch $makeException
 953           // if note field is subject then return subject else body of the note
 954           $noteColumn = 'note';
 955           if (isset($noteField) && $noteField === 'note_subject') {
 956             $noteColumn = 'subject';
 957           }
 958
 959           $this->_useGroupBy = TRUE;
 960           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 961           $this->_element[$name] = 1;
 962           $this->_tables['civicrm_note'] = 1;
 963         }
 964         elseif ($name === 'current_employer') {
 965           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 966           $this->_element[$name] = 1;
 967         }
 968       }
 969
 970       if ($cfID && !empty($field['is_search_range'])) {
 971         // this is a custom field with range search enabled, so we better check for two/from values
 972         if (!empty($this->_paramLookup[$name . '_from'])) {
 973           if (!array_key_exists($cfID, $this->_cfIDs)) {
 974             $this->_cfIDs[$cfID] = [];
 975           }
 976           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 977             // search in the cdID array for the same grouping
 978             $fnd = FALSE;
 979             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 980               if ($c[3] == $p[3]) {
 981                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 982                 $fnd = TRUE;
 983               }
 984             }
 985             if (!$fnd) {
 986               $p[2] = ['from' => $p[2]];
 987               $this->_cfIDs[$cfID][] = $p;
 988             }
 989           }
 990         }
 991         if (!empty($this->_paramLookup[$name . '_to'])) {
 992           if (!array_key_exists($cfID, $this->_cfIDs)) {
 993             $this->_cfIDs[$cfID] = [];
 994           }
 995           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
 996             // search in the cdID array for the same grouping
 997             $fnd = FALSE;
 998             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 999               if ($c[4] == $p[4]) {
1000                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1001                 $fnd = TRUE;
1002               }
1003             }
1004             if (!$fnd) {
1005               $p[2] = ['to' => $p[2]];
1006               $this->_cfIDs[$cfID][] = $p;
1007             }
1008           }
1009         }
1010       }
1011     }
1012
1013     // add location as hierarchical elements
1014     $this->addHierarchicalElements();
1015
1016     // add multiple field like website
1017     $this->addMultipleElements();
1018
1019     //fix for CRM-951
1020     CRM_Core_Component::alterQuery($this, 'select');
1021
1022     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1023   }
1024
1025   /**
1026    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1027    */
1028   public function addHierarchicalElements() {
1029     if (empty($this->_returnProperties['location'])) {
1030       return;
1031     }
1032     if (!is_array($this->_returnProperties['location'])) {
1033       return;
1034     }
1035
1036     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1037     $processed = [];
1038     $index = 0;
1039
1040     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1041     $addressCustomFieldIds = [];
1042
1043     foreach ($this->_returnProperties['location'] as $name => $elements) {
1044       $lCond = self::getPrimaryCondition($name);
1045       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1046
1047       if (!$lCond) {
1048         if ($locationTypeId === FALSE) {
1049           continue;
1050         }
1051         $lCond = "location_type_id = $locationTypeId";
1052         $this->_useDistinct = TRUE;
1053
1054         //commented for CRM-3256
1055         $this->_useGroupBy = TRUE;
1056       }
1057
1058       $name = str_replace(' ', '_', $name);
1059       $tName = "$name-location_type";
1060       $ltName = "`$name-location_type`";
1061       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1062       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1063       $this->_element["{$tName}_id"] = 1;
1064       $this->_element["{$tName}"] = 1;
1065
1066       $locationTypeName = $tName;
1067       $locationTypeJoin = [];
1068
1069       $addWhereCount = 0;
1070       foreach ($elements as $elementFullName => $dontCare) {
1071         $index++;
1072         $elementName = $elementCmpName = $elementFullName;
1073
1074         if (substr($elementCmpName, 0, 5) == 'phone') {
1075           $elementCmpName = 'phone';
1076         }
1077
1078         if (in_array($elementCmpName, array_keys($addressCustomFields))) {
1079           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1080             $addressCustomFieldIds[$cfID][$name] = 1;
1081           }
1082         }
1083         // add address table - doesn't matter if we do it mutliple times - it's the same data
1084         // @todo ditch the double processing of addressJoin
1085         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1086           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1087         ) {
1088           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1089           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1090           $processed[$aName] = 1;
1091         }
1092
1093         $cond = $elementType = '';
1094         if (strpos($elementName, '-') !== FALSE) {
1095           // this is either phone, email or IM
1096           list($elementName, $elementType) = explode('-', $elementName);
1097
1098           if (($elementName != 'phone') && ($elementName != 'im')) {
1099             $cond = self::getPrimaryCondition($elementType);
1100           }
1101           // CRM-13011 : If location type is primary, do not restrict search to the phone
1102           // type id - we want the primary phone, regardless of what type it is.
1103           // Otherwise, restrict to the specified phone type for the given field.
1104           if ((!$cond) && ($elementName == 'phone')) {
1105             $cond = "phone_type_id = '$elementType'";
1106           }
1107           elseif ((!$cond) && ($elementName == 'im')) {
1108             // IM service provider id, CRM-3140
1109             $cond = "provider_id = '$elementType'";
1110           }
1111           $elementType = '-' . $elementType;
1112         }
1113
1114         $field = $this->_fields[$elementName] ?? NULL;
1115         if (isset($this->_pseudoConstantsSelect[$field['name']])) {
1116           $this->_pseudoConstantsSelect[$name . '-' . $field['name']] = $this->_pseudoConstantsSelect[$field['name']];
1117         }
1118
1119         // hack for profile, add location id
1120         if (!$field) {
1121           if ($elementType &&
1122             // fix for CRM-882( to handle phone types )
1123             !is_numeric($elementType)
1124           ) {
1125             if (is_numeric($name)) {
1126               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1127             }
1128             else {
1129               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1130             }
1131           }
1132           elseif (is_numeric($name)) {
1133             //this for phone type to work
1134             if (in_array($elementName, ['phone', 'phone_ext'])) {
1135               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1136             }
1137             else {
1138               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1139             }
1140           }
1141           else {
1142             //this is for phone type to work for profile edit
1143             if (in_array($elementName, ['phone', 'phone_ext'])) {
1144               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1145             }
1146             else {
1147               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1148             }
1149           }
1150         }
1151
1152         // Check if there is a value, if so also add to where Clause
1153         $addWhere = FALSE;
1154         if ($this->_params) {
1155           $nm = $elementName;
1156           if (isset($locationTypeId)) {
1157             $nm .= "-$locationTypeId";
1158           }
1159           if (!is_numeric($elementType)) {
1160             $nm .= "$elementType";
1161           }
1162
1163           foreach ($this->_params as $id => $values) {
1164             if ((is_array($values) && $values[0] == $nm) ||
1165               (in_array($elementName, ['phone', 'im'])
1166                 && (strpos($values[0], $nm) !== FALSE)
1167               )
1168             ) {
1169               $addWhere = TRUE;
1170               $addWhereCount++;
1171               break;
1172             }
1173           }
1174         }
1175
1176         if ($field && isset($field['where'])) {
1177           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1178           $pf = substr($tableName, 8);
1179           $tName = $name . '-' . $pf . $elementType;
1180           if (isset($tableName)) {
1181             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1182               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1183             }
1184             else {
1185               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1186             }
1187
1188             $this->_element["{$tName}_id"] = 1;
1189             if (substr($tName, -15) == '-state_province') {
1190               // FIXME: hack to fix CRM-1900
1191               $a = Civi::settings()->get('address_format');
1192
1193               if (substr_count($a, 'state_province_name') > 0) {
1194                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1195                   'pseudoField' => "{$pf}_id",
1196                   'idCol' => "{$tName}_id",
1197                   'bao' => 'CRM_Core_BAO_Address',
1198                 ];
1199                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1200               }
1201               else {
1202                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1203                   'pseudoField' => 'state_province_abbreviation',
1204                   'idCol' => "{$tName}_id",
1205                 ];
1206                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1207               }
1208             }
1209             else {
1210               if (substr($elementFullName, 0, 2) == 'im') {
1211                 $provider = "{$name}-{$elementFullName}-provider_id";
1212                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1213                 $this->_element[$provider] = 1;
1214               }
1215               if ($pf == 'country' || $pf == 'county') {
1216                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1217                   'pseudoField' => "{$pf}_id",
1218                   'idCol' => "{$tName}_id",
1219                   'bao' => 'CRM_Core_BAO_Address',
1220                 ];
1221                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1222               }
1223               else {
1224                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1225               }
1226             }
1227
1228             if (in_array($pf, ['state_province', 'country', 'county'])) {
1229               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1230             }
1231             else {
1232               $this->_element["{$name}-{$elementFullName}"] = 1;
1233             }
1234
1235             if (empty($processed["`$tName`"])) {
1236               $processed["`$tName`"] = 1;
1237               $newName = $tableName . '_' . $index;
1238               switch ($tableName) {
1239                 case 'civicrm_phone':
1240                 case 'civicrm_email':
1241                 case 'civicrm_im':
1242                 case 'civicrm_openid':
1243
1244                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1245                   if ($tableName != 'civicrm_phone') {
1246                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1247                   }
1248                   elseif (is_numeric($name)) {
1249                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1250                   }
1251
1252                   // this special case to add phone type
1253                   if ($cond) {
1254                     $phoneTypeCondition = " AND `$tName`.$cond ";
1255                     //gross hack to pickup corrupted data also, CRM-7603
1256                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1257                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1258                       if (!empty($lCond)) {
1259                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1260                       }
1261                     }
1262                     $this->_tables[$tName] .= $phoneTypeCondition;
1263                   }
1264
1265                   //build locationType join
1266                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1267
1268                   if ($addWhere) {
1269                     $this->_whereTables[$tName] = $this->_tables[$tName];
1270                   }
1271                   break;
1272
1273                 case 'civicrm_state_province':
1274                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1275                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1276                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1277                   if ($addWhere) {
1278                     $this->_whereTables["{$name}-address"] = $addressJoin;
1279                   }
1280                   break;
1281
1282                 case 'civicrm_country':
1283                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1284                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1285                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1286                   if ($addWhere) {
1287                     $this->_whereTables["{$name}-address"] = $addressJoin;
1288                   }
1289                   break;
1290
1291                 case 'civicrm_county':
1292                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1293                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1294                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1295                   if ($addWhere) {
1296                     $this->_whereTables["{$name}-address"] = $addressJoin;
1297                   }
1298                   break;
1299
1300                 default:
1301                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1302                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1303                   }
1304                   if ($addWhere) {
1305                     $this->_whereTables["{$name}-address"] = $addressJoin;
1306                   }
1307                   break;
1308               }
1309             }
1310           }
1311         }
1312       }
1313
1314       // add location type  join
1315       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1316       $this->_tables[$locationTypeName] = $ltypeJoin;
1317
1318       // table should be present in $this->_whereTables,
1319       // to add its condition in location type join, CRM-3939.
1320       if ($addWhereCount) {
1321         $locClause = [];
1322         foreach ($this->_whereTables as $tableName => $clause) {
1323           if (!empty($locationTypeJoin[$tableName])) {
1324             $locClause[] = $locationTypeJoin[$tableName];
1325           }
1326         }
1327
1328         if (!empty($locClause)) {
1329           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1330         }
1331       }
1332     }
1333
1334     if (!empty($addressCustomFieldIds)) {
1335       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1336       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1337         foreach ($locTypeName as $name => $dnc) {
1338           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1339           $fieldName = "$name-custom_{$cfID}";
1340           $tName = "$name-address-custom-{$cfID}";
1341           $aName = "`$name-address-custom-{$cfID}`";
1342           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1343           $this->_element["{$tName}_id"] = 1;
1344           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1345           $this->_element[$fieldName] = 1;
1346           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1347         }
1348       }
1349     }
1350   }
1351
1352   /**
1353    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1354    */
1355   public function addMultipleElements() {
1356     if (empty($this->_returnProperties['website'])) {
1357       return;
1358     }
1359     if (!is_array($this->_returnProperties['website'])) {
1360       return;
1361     }
1362
1363     foreach ($this->_returnProperties['website'] as $key => $elements) {
1364       foreach ($elements as $elementFullName => $dontCare) {
1365         $tName = "website-{$key}-{$elementFullName}";
1366         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1367         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1368         $this->_element["{$tName}_id"] = 1;
1369         $this->_element["{$tName}"] = 1;
1370
1371         $type = "website-{$key}-website_type_id";
1372         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1373         $this->_element[$type] = 1;
1374         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1375       }
1376     }
1377   }
1378
1379   /**
1380    * Generate the query based on what type of query we need.
1381    *
1382    * @param bool $count
1383    * @param bool $sortByChar
1384    * @param bool $groupContacts
1385    * @param bool $onlyDeleted
1386    *
1387    * @return array
1388    *   sql query parts as an array
1389    */
1390   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1391     // build permission clause
1392     $this->generatePermissionClause($onlyDeleted, $count);
1393
1394     if ($count) {
1395       if (isset($this->_rowCountClause)) {
1396         $select = "SELECT {$this->_rowCountClause}";
1397       }
1398       elseif (isset($this->_distinctComponentClause)) {
1399         // we add distinct to get the right count for components
1400         // for the more complex result set, we use GROUP BY the same id
1401         // CRM-9630
1402         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1403       }
1404       else {
1405         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1406       }
1407       $from = $this->_simpleFromClause;
1408       if ($this->_useDistinct) {
1409         $this->_useGroupBy = TRUE;
1410       }
1411     }
1412     elseif ($sortByChar) {
1413       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1414       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1415       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1416       $from = $this->_simpleFromClause;
1417     }
1418     elseif ($groupContacts) {
1419       $select = 'SELECT contact_a.id as id';
1420       if ($this->_useDistinct) {
1421         $this->_useGroupBy = TRUE;
1422       }
1423       $from = $this->_simpleFromClause;
1424     }
1425     else {
1426       if (!empty($this->_paramLookup['group'])) {
1427
1428         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1429
1430         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1431           $this->_paramLookup['group'][0][1] = key($value);
1432         }
1433
1434         // Presumably the lines below come into manage groups screen.
1435         // make sure there is only one element
1436         // this is used when we are running under smog and need to know
1437         // how the contact was added (CRM-1203)
1438         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1439         if ((count($this->_paramLookup['group']) == 1) &&
1440           (count($groups) == 1)
1441         ) {
1442           $groupId = $groups[0];
1443
1444           //check if group is saved search
1445           $group = new CRM_Contact_BAO_Group();
1446           $group->id = $groupId;
1447           $group->find(TRUE);
1448
1449           if (!isset($group->saved_search_id)) {
1450             $tbName = "civicrm_group_contact";
1451             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1452             // as this will add load to an intentionally light query.
1453             // ideally this code would be removed as it appears to be to support CRM-1203
1454             // and passing in the required returnProperties from the url would
1455             // make more sense that globally applying the requirements of one form.
1456             if (($this->_returnProperties != ['contact_id'])) {
1457               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1458               $this->_element['group_contact_id'] = 1;
1459               $this->_select['status'] = "$tbName.status as status";
1460               $this->_element['status'] = 1;
1461             }
1462           }
1463         }
1464         $this->_useGroupBy = TRUE;
1465       }
1466       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1467         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1468           // CRM-5954
1469           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1470           $this->_useDistinct = FALSE;
1471           $this->_useGroupBy = TRUE;
1472         }
1473       }
1474
1475       $select = $this->getSelect();
1476       $from = $this->_fromClause;
1477     }
1478
1479     $where = '';
1480     if (!empty($this->_whereClause)) {
1481       $where = "WHERE {$this->_whereClause}";
1482     }
1483
1484     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1485       if (!empty($this->_permissionFromClause)) {
1486         $from .= " $this->_permissionFromClause";
1487       }
1488       if (empty($where)) {
1489         $where = "WHERE $this->_permissionWhereClause";
1490       }
1491       else {
1492         $where = "$where AND $this->_permissionWhereClause";
1493       }
1494     }
1495
1496     $having = '';
1497     if (!empty($this->_having)) {
1498       foreach ($this->_having as $havingSets) {
1499         foreach ($havingSets as $havingSet) {
1500           $havingValue[] = $havingSet;
1501         }
1502       }
1503       $having = ' HAVING ' . implode(' AND ', $havingValue);
1504     }
1505
1506     // if we are doing a transform, do it here
1507     // use the $from, $where and $having to get the contact ID
1508     if ($this->_displayRelationshipType) {
1509       $this->filterRelatedContacts($from, $where, $having);
1510     }
1511
1512     return [$select, $from, $where, $having];
1513   }
1514
1515   /**
1516    * Get where values from the parameters.
1517    *
1518    * @param string $name
1519    * @param mixed $grouping
1520    *
1521    * @return mixed
1522    */
1523   public function getWhereValues($name, $grouping) {
1524     $result = NULL;
1525     foreach ($this->_params as $values) {
1526       if ($values[0] == $name && $values[3] == $grouping) {
1527         return $values;
1528       }
1529     }
1530
1531     return $result;
1532   }
1533
1534   /**
1535    * Fix date values.
1536    *
1537    * @param bool $relative
1538    * @param string $from
1539    * @param string $to
1540    */
1541   public static function fixDateValues($relative, &$from, &$to) {
1542     if ($relative) {
1543       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1544     }
1545   }
1546
1547   /**
1548    * Convert values from form-appropriate to query-object appropriate.
1549    *
1550    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1551    * So, ideally we would convert all fields to look like
1552    *  array(
1553    *   0 => $fieldName
1554    *   // Set the operator for legacy reasons, but it is ignored
1555    *   1 =>  '='
1556    *   // array in sql filter syntax
1557    *   2 => array('BETWEEN' => array(1,60),
1558    *   3 => null
1559    *   4 => null
1560    *  );
1561    *
1562    * There are some examples of the syntax in
1563    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1564    *
1565    * More notes at CRM_Core_DAO::createSQLFilter
1566    *
1567    * and a list of supported operators in CRM_Core_DAO
1568    *
1569    * @param array $formValues
1570    * @param int $wildcard
1571    * @param bool $useEquals
1572    *
1573    * @param string $apiEntity
1574    *
1575    * @param array $entityReferenceFields
1576    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1577    *
1578    * @return array
1579    */
1580   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1581     $entityReferenceFields = []) {
1582     $params = [];
1583     if (empty($formValues)) {
1584       return $params;
1585     }
1586
1587     self::filterCountryFromValuesIfStateExists($formValues);
1588     CRM_Core_BAO_CustomValue::fixCustomFieldValue($formValues);
1589
1590     foreach ($formValues as $id => $values) {
1591       if (self::isAlreadyProcessedForQueryFormat($values)) {
1592         $params[] = $values;
1593         continue;
1594       }
1595
1596       self::legacyConvertFormValues($id, $values);
1597
1598       // The form uses 1 field to represent two db fields
1599       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1600         $contactType = [];
1601         $subType = [];
1602         foreach ((array) $values as $key => $type) {
1603           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1604           $contactType[$types[0]] = $types[0];
1605           // Add sub-type if specified
1606           if (!empty($types[1])) {
1607             $subType[$types[1]] = $types[1];
1608           }
1609         }
1610         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1611         if ($subType) {
1612           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1613         }
1614       }
1615       elseif ($id === 'privacy') {
1616         if (is_array($formValues['privacy'])) {
1617           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1618           foreach ($formValues['privacy'] as $key => $value) {
1619             if ($value) {
1620               $params[] = [$key, $op, $value, 0, 0];
1621             }
1622           }
1623         }
1624       }
1625       elseif ($id === 'email_on_hold') {
1626         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1627           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1628           // so in 5.11 we have an extra if that should become redundant over time.
1629           // https://lab.civicrm.org/dev/core/issues/745
1630           // @todo this renaming of email_on_hold to on_hold needs revisiting
1631           // it precedes recent changes but causes the default not to reload.
1632           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1633           if (!empty($onHoldValue)) {
1634             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1635           }
1636         }
1637       }
1638       elseif (substr($id, 0, 7) === 'custom_'
1639         &&  (
1640           substr($id, -5, 5) === '_from'
1641           || substr($id, -3, 3) === '_to'
1642         )
1643       ) {
1644         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1645       }
1646       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1647         FALSE)) {
1648         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1649       }
1650       else {
1651         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1652
1653         if (!$values) {
1654           continue;
1655         }
1656         $params[] = $values;
1657       }
1658     }
1659     return $params;
1660   }
1661
1662   /**
1663    * Function to support legacy format for groups and tags.
1664    *
1665    * @param string $id
1666    * @param array|int $values
1667    *
1668    */
1669   public static function legacyConvertFormValues($id, &$values) {
1670     $legacyElements = [
1671       'group',
1672       'tag',
1673       'contact_tags',
1674       'contact_type',
1675       'membership_type_id',
1676       'membership_status_id',
1677     ];
1678     if (in_array($id, $legacyElements) && is_array($values)) {
1679       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1680       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1681       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1682     }
1683   }
1684
1685   /**
1686    * Fix values from query from/to something no-one cared enough to document.
1687    *
1688    * @param int $id
1689    * @param array $values
1690    * @param int $wildcard
1691    * @param bool $useEquals
1692    *
1693    * @param string $apiEntity
1694    *
1695    * @return array|null
1696    */
1697   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1698     // skip a few search variables
1699     static $skipWhere = NULL;
1700     static $likeNames = NULL;
1701     $result = NULL;
1702
1703     // Change camelCase EntityName to lowercase with underscores
1704     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1705
1706     // check if $value is in OK (Operator as Key) format as used by Get API
1707     if (CRM_Utils_System::isNull($values)) {
1708       return $result;
1709     }
1710
1711     if (!$skipWhere) {
1712       $skipWhere = [
1713         'task',
1714         'radio_ts',
1715         'uf_group_id',
1716         'component_mode',
1717         'qfKey',
1718         'operator',
1719         'display_relationship_type',
1720       ];
1721     }
1722
1723     if (in_array($id, $skipWhere) ||
1724       substr($id, 0, 4) == '_qf_' ||
1725       substr($id, 0, 7) == 'hidden_'
1726     ) {
1727       return $result;
1728     }
1729
1730     if ($apiEntity &&
1731       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1732       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1733       (substr($id, 0, 7) != 'custom_')
1734     ) {
1735       $id = $apiEntity . '_' . $id;
1736     }
1737
1738     if (!$likeNames) {
1739       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1740     }
1741
1742     // email comes in via advanced search
1743     // so use wildcard always
1744     if ($id == 'email') {
1745       $wildcard = 1;
1746     }
1747
1748     if (!$useEquals && in_array($id, $likeNames)) {
1749       $result = [$id, 'LIKE', $values, 0, 1];
1750     }
1751     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1752       $result = [$id, 'LIKE', $values, 0, 0];
1753     }
1754     elseif ($id == 'contact_type' ||
1755       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1756     ) {
1757       $result = [$id, 'IN', $values, 0, $wildcard];
1758     }
1759     else {
1760       $result = [$id, '=', $values, 0, $wildcard];
1761     }
1762
1763     return $result;
1764   }
1765
1766   /**
1767    * Get the where clause for a single field.
1768    *
1769    * @param array $values
1770    * @param bool $isForcePrimaryOnly
1771    *
1772    * @throws \CRM_Core_Exception
1773    */
1774   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1775     if ($this->isARelativeDateField($values[0])) {
1776       $this->buildRelativeDateQuery($values);
1777       return;
1778     }
1779     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1780
1781     // do not process custom fields or prefixed contact ids or component params
1782     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1783       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1784       (substr($values[0], 0, 13) === 'contribution_') ||
1785       (substr($values[0], 0, 6) === 'event_') ||
1786       (substr($values[0], 0, 12) === 'participant_') ||
1787       (substr($values[0], 0, 7) === 'member_') ||
1788       (substr($values[0], 0, 6) === 'grant_') ||
1789       (substr($values[0], 0, 7) === 'pledge_') ||
1790       (substr($values[0], 0, 5) === 'case_') ||
1791       (substr($values[0], 0, 10) === 'financial_') ||
1792       (substr($values[0], 0, 8) === 'payment_') ||
1793       (substr($values[0], 0, 11) === 'membership_')
1794       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1795       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1796       // be needed.
1797       || $values[0] === 'mailing_id'
1798     ) {
1799       return;
1800     }
1801
1802     // skip for hook injected fields / params
1803     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1804     if (array_key_exists($values[0], $extFields)) {
1805       return;
1806     }
1807
1808     switch ($values[0]) {
1809       case 'deleted_contacts':
1810         $this->deletedContacts($values);
1811         return;
1812
1813       case 'contact_sub_type':
1814         $this->contactSubType($values);
1815         return;
1816
1817       case 'group':
1818       case 'group_type':
1819         $this->group($values);
1820         return;
1821
1822       // case tag comes from find contacts
1823       case 'tag_search':
1824         $this->tagSearch($values);
1825         return;
1826
1827       case 'tag':
1828       case 'contact_tags':
1829         $this->tag($values);
1830         return;
1831
1832       case 'note':
1833       case 'note_body':
1834       case 'note_subject':
1835         $this->notes($values);
1836         return;
1837
1838       case 'uf_user':
1839         $this->ufUser($values);
1840         return;
1841
1842       case 'sort_name':
1843       case 'display_name':
1844         $this->sortName($values);
1845         return;
1846
1847       case 'addressee':
1848       case 'postal_greeting':
1849       case 'email_greeting':
1850         $this->greetings($values);
1851         return;
1852
1853       case 'email':
1854       case 'email_id':
1855         $this->email($values, $isForcePrimaryOnly);
1856         return;
1857
1858       case 'phone_numeric':
1859         $this->phone_numeric($values);
1860         return;
1861
1862       case 'phone_phone_type_id':
1863       case 'phone_location_type_id':
1864         $this->phone_option_group($values);
1865         return;
1866
1867       case 'street_address':
1868         $this->street_address($values);
1869         return;
1870
1871       case 'street_number':
1872         $this->street_number($values);
1873         return;
1874
1875       case 'sortByCharacter':
1876         $this->sortByCharacter($values);
1877         return;
1878
1879       case 'location_type':
1880         $this->locationType($values);
1881         return;
1882
1883       case 'county':
1884         $this->county($values);
1885         return;
1886
1887       case 'state_province':
1888       case 'state_province_id':
1889       case 'state_province_name':
1890         $this->stateProvince($values);
1891         return;
1892
1893       case 'country':
1894       case 'country_id':
1895         $this->country($values, FALSE);
1896         return;
1897
1898       case 'postal_code':
1899       case 'postal_code_low':
1900       case 'postal_code_high':
1901         $this->postalCode($values);
1902         return;
1903
1904       case 'activity_date':
1905       case 'activity_date_low':
1906       case 'activity_date_high':
1907       case 'activity_date_time_low':
1908       case 'activity_date_time_high':
1909       case 'activity_role':
1910       case 'activity_status_id':
1911       case 'activity_status':
1912       case 'activity_priority':
1913       case 'activity_priority_id':
1914       case 'followup_parent_id':
1915       case 'parent_id':
1916       case 'source_contact_id':
1917       case 'activity_text':
1918       case 'activity_option':
1919       case 'test_activities':
1920       case 'activity_type_id':
1921       case 'activity_type':
1922       case 'activity_survey_id':
1923       case 'activity_tags':
1924       case 'activity_taglist':
1925       case 'activity_test':
1926       case 'activity_campaign_id':
1927       case 'activity_engagement_level':
1928       case 'activity_id':
1929       case 'activity_result':
1930       case 'source_contact':
1931         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1932         return;
1933
1934       case 'age_low':
1935       case 'age_high':
1936       case 'birth_date_low':
1937       case 'birth_date_high':
1938       case 'deceased_date_low':
1939       case 'deceased_date_high':
1940         $this->demographics($values);
1941         return;
1942
1943       case 'age_asof_date':
1944         // handled by demographics
1945         return;
1946
1947       case 'log_date_low':
1948       case 'log_date_high':
1949         $this->modifiedDates($values);
1950         return;
1951
1952       case 'changed_by':
1953         $this->changeLog($values);
1954         return;
1955
1956       case 'do_not_phone':
1957       case 'do_not_email':
1958       case 'do_not_mail':
1959       case 'do_not_sms':
1960       case 'do_not_trade':
1961       case 'is_opt_out':
1962         $this->privacy($values);
1963         return;
1964
1965       case 'privacy_options':
1966         $this->privacyOptions($values);
1967         return;
1968
1969       case 'privacy_operator':
1970       case 'privacy_toggle':
1971         // these are handled by privacy options
1972         return;
1973
1974       case 'preferred_communication_method':
1975         $this->preferredCommunication($values);
1976         return;
1977
1978       case 'relation_type_id':
1979       case 'relationship_start_date_high':
1980       case 'relationship_start_date_low':
1981       case 'relationship_end_date_high':
1982       case 'relationship_end_date_low':
1983       case 'relation_active_period_date_high':
1984       case 'relation_active_period_date_low':
1985       case 'relation_target_name':
1986       case 'relation_status':
1987       case 'relation_description':
1988       case 'relation_date_low':
1989       case 'relation_date_high':
1990         $this->relationship($values);
1991         $this->_relationshipValuesAdded = TRUE;
1992         return;
1993
1994       case 'task_status_id':
1995         $this->task($values);
1996         return;
1997
1998       case 'task_id':
1999         // since this case is handled with the above
2000         return;
2001
2002       case 'prox_distance':
2003         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2004         return;
2005
2006       case 'prox_street_address':
2007       case 'prox_city':
2008       case 'prox_postal_code':
2009       case 'prox_state_province_id':
2010       case 'prox_country_id':
2011       case 'prox_geo_code_1':
2012       case 'prox_geo_code_2':
2013         // handled by the proximity_distance clause
2014         return;
2015
2016       default:
2017         $this->restWhere($values);
2018         return;
2019     }
2020   }
2021
2022   /**
2023    * Given a list of conditions in params generate the required where clause.
2024    *
2025    * @param bool $isForcePrimaryEmailOnly
2026    *
2027    * @return string
2028    * @throws \CRM_Core_Exception
2029    */
2030   public function whereClause($isForcePrimaryEmailOnly = NULL) {
2031     $this->_where[0] = [];
2032     $this->_qill[0] = [];
2033
2034     $this->includeContactIDs();
2035     if (!empty($this->_params)) {
2036       foreach (array_keys($this->_params) as $id) {
2037         if (empty($this->_params[$id][0])) {
2038           continue;
2039         }
2040         // check for both id and contact_id
2041         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2042           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2043           $field = $this->_fields['id'] ?? NULL;
2044           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2045             'CRM_Contact_BAO_Contact',
2046             "contact_a.id",
2047             $this->_params[$id][2],
2048             $this->_params[$id][1]
2049           );
2050           $this->_qill[0][] = ts("%1 %2 %3", [
2051             1 => $field['title'] ?? '',
2052             2 => $qillop,
2053             3 => $qillVal,
2054           ]);
2055         }
2056         else {
2057           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2058         }
2059       }
2060
2061       CRM_Core_Component::alterQuery($this, 'where');
2062
2063       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2064     }
2065
2066     if ($this->_customQuery) {
2067       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2068       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2069       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2070       if (!empty($this->_customQuery->_where)) {
2071         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2072       }
2073       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2074     }
2075
2076     $clauses = [];
2077     $andClauses = [];
2078
2079     $validClauses = 0;
2080     if (!empty($this->_where)) {
2081       foreach ($this->_where as $grouping => $values) {
2082         if ($grouping > 0 && !empty($values)) {
2083           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2084           $validClauses++;
2085         }
2086       }
2087
2088       if (!empty($this->_where[0])) {
2089         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2090       }
2091       if (!empty($clauses)) {
2092         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2093       }
2094
2095       if ($validClauses > 1) {
2096         $this->_useDistinct = TRUE;
2097       }
2098     }
2099
2100     return implode(' AND ', $andClauses);
2101   }
2102
2103   /**
2104    * Generate where clause for any parameters not already handled.
2105    *
2106    * @param array $values
2107    *
2108    * @throws Exception
2109    */
2110   public function restWhere(&$values) {
2111     $name = $values[0] ?? NULL;
2112     $op = $values[1] ?? NULL;
2113     $value = $values[2] ?? NULL;
2114     $grouping = $values[3] ?? NULL;
2115     $wildcard = $values[4] ?? NULL;
2116
2117     if (isset($grouping) && empty($this->_where[$grouping])) {
2118       $this->_where[$grouping] = [];
2119     }
2120
2121     $multipleFields = ['url'];
2122
2123     //check if the location type exists for fields
2124     $lType = '';
2125     $locType = explode('-', $name);
2126
2127     if (!in_array($locType[0], $multipleFields)) {
2128       //add phone type if exists
2129       if (isset($locType[2]) && $locType[2]) {
2130         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2131       }
2132     }
2133
2134     $field = $this->_fields[$name] ?? NULL;
2135
2136     if (!$field) {
2137       $field = $this->_fields[$locType[0]] ?? NULL;
2138
2139       if (!$field) {
2140         // Strip any trailing _high & _low that might be appended.
2141         $realFieldName = str_replace(['_high', '_low'], '', $name);
2142         if (isset($this->_fields[$realFieldName])) {
2143           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2144           $columnName = $field['column_name'] ?? $field['name'];
2145           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2146         }
2147         return;
2148       }
2149     }
2150
2151     $setTables = TRUE;
2152
2153     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2154     if (isset($locType[1]) && is_numeric($locType[1])) {
2155       $lType = $locationType[$locType[1]];
2156     }
2157     if ($lType) {
2158       $field['title'] .= " ($lType)";
2159     }
2160
2161     if (substr($name, 0, 14) === 'state_province') {
2162       if (isset($locType[1]) && is_numeric($locType[1])) {
2163         $setTables = FALSE;
2164         $aName = "{$lType}-address";
2165         $where = "`$aName`.state_province_id";
2166       }
2167       else {
2168         $where = "civicrm_address.state_province_id";
2169       }
2170
2171       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2172       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2173       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2174       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2175     }
2176     elseif (!empty($field['pseudoconstant'])) {
2177       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2178       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2179       $this->optionValueQuery(
2180         $name, $op, $value, $grouping,
2181         'CRM_Contact_DAO_Contact',
2182         $field,
2183         $field['html']['label'] ?? $field['title'],
2184         CRM_Utils_Type::typeToString($dataType)
2185       );
2186       if ($name === 'gender_id') {
2187         self::$_openedPanes[ts('Demographics')] = TRUE;
2188       }
2189     }
2190     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2191       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2192       if (isset($locType[1]) && is_numeric($locType[1])) {
2193         $setTables = FALSE;
2194         $aName = "{$lType}-address";
2195         $where = "`$aName`.$name";
2196       }
2197       else {
2198         $where = "civicrm_address.$name";
2199       }
2200
2201       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2202       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2203
2204       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2205       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2206     }
2207     elseif ($name === 'world_region') {
2208       $this->optionValueQuery(
2209         $name, $op, $value, $grouping,
2210         NULL,
2211         $field,
2212         ts('World Region'),
2213         'Positive'
2214       );
2215     }
2216     elseif ($name === 'is_deceased') {
2217       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2218       self::$_openedPanes[ts('Demographics')] = TRUE;
2219     }
2220     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2221       $appendDateTime = TRUE;
2222       if ($name === 'deceased_date' || $name === 'birth_date') {
2223         $appendDateTime = FALSE;
2224         self::$_openedPanes[ts('Demographics')] = TRUE;
2225       }
2226       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2227     }
2228     elseif ($name === 'contact_id') {
2229       if (is_int($value)) {
2230         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2231         $this->_qill[$grouping][] = "$field[title] $op $value";
2232       }
2233     }
2234     elseif ($name === 'name') {
2235       $value = CRM_Core_DAO::escapeString($value);
2236       if ($wildcard) {
2237         $op = 'LIKE';
2238         $value = self::getWildCardedValue($wildcard, $op, $value);
2239       }
2240       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2241       // @todo it's likely this code path is obsolete / never called. It is definitely not
2242       // passed through in our test suite.
2243       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2244       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2245     }
2246     elseif ($name === 'current_employer') {
2247       if ($wildcard) {
2248         $op = 'LIKE';
2249         $value = self::getWildCardedValue($wildcard, $op, $value);
2250       }
2251       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2252         $value
2253       );
2254       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2255       $this->_where[$grouping][] = $ceWhereClause;
2256       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2257     }
2258     elseif (substr($name, 0, 4) === 'url-') {
2259       $tName = 'civicrm_website';
2260       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2261       $value = CRM_Core_DAO::escapeString($value);
2262       if ($wildcard) {
2263         $op = 'LIKE';
2264         $value = self::getWildCardedValue($wildcard, $op, $value);
2265       }
2266
2267       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2268       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2269     }
2270     elseif ($name === 'contact_is_deleted') {
2271       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2272     }
2273     elseif (!empty($field['where'])) {
2274       $type = NULL;
2275       if (!empty($field['type'])) {
2276         $type = CRM_Utils_Type::typeToString($field['type']);
2277       }
2278
2279       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2280
2281       if (isset($locType[1]) &&
2282         is_numeric($locType[1])
2283       ) {
2284         $setTables = FALSE;
2285
2286         //get the location name
2287         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2288         $fieldName = "`$tName`.$fldName";
2289
2290         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2291         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2292
2293       }
2294       else {
2295         if ($tableName == 'civicrm_contact') {
2296           $fieldName = "contact_a.{$fieldName}";
2297         }
2298         else {
2299           $fieldName = $field['where'];
2300         }
2301       }
2302
2303       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2304       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2305         1 => $field['title'],
2306         2 => $qillop,
2307         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2308       ]);
2309
2310       if (is_array($value)) {
2311         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2312         // for api style operators for functions that hit this point without worrying about regression
2313         // (the previous comments indicated the condition for hitting this point were unknown
2314         // per CRM-14743 we are adding modified_date & created_date operator support
2315         $operations = array_keys($value);
2316         foreach ($operations as $operator) {
2317           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2318             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2319             //so this condition will satisfy the search for now
2320             if (strpos($op, 'IN') !== FALSE) {
2321               $value = [$op => $value];
2322             }
2323             // we don't know when this might happen
2324             else {
2325               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2326             }
2327           }
2328         }
2329         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2330       }
2331       else {
2332         if ($wildcard) {
2333           $op = 'LIKE';
2334           $value = self::getWildCardedValue($wildcard, $op, $value);
2335         }
2336
2337         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2338       }
2339     }
2340
2341     if ($setTables && isset($field['where'])) {
2342       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2343       if (isset($tableName)) {
2344         $this->_tables[$tableName] = 1;
2345         $this->_whereTables[$tableName] = 1;
2346       }
2347     }
2348   }
2349
2350   /**
2351    * @param $where
2352    * @param $locType
2353    *
2354    * @return array
2355    * @throws Exception
2356    */
2357   public static function getLocationTableName(&$where, &$locType) {
2358     if (isset($locType[1]) && is_numeric($locType[1])) {
2359       list($tbName, $fldName) = explode(".", $where);
2360
2361       //get the location name
2362       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2363       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2364       if (in_array($locType[0], $specialFields)) {
2365         //hack to fix / special handing for phone_ext
2366         if ($locType[0] == 'phone_ext') {
2367           $locType[0] = 'phone';
2368         }
2369         if (isset($locType[2]) && $locType[2]) {
2370           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2371         }
2372         else {
2373           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2374         }
2375       }
2376       elseif (in_array($locType[0],
2377         [
2378           'address_name',
2379           'street_address',
2380           'street_name',
2381           'street_number_suffix',
2382           'street_unit',
2383           'supplemental_address_1',
2384           'supplemental_address_2',
2385           'supplemental_address_3',
2386           'city',
2387           'postal_code',
2388           'postal_code_suffix',
2389           'geo_code_1',
2390           'geo_code_2',
2391           'master_id',
2392         ]
2393       )) {
2394         //fix for search by profile with address fields.
2395         $tName = "{$locationType[$locType[1]]}-address";
2396       }
2397       elseif (in_array($locType[0],
2398           [
2399             'on_hold',
2400             'signature_html',
2401             'signature_text',
2402             'is_bulkmail',
2403           ]
2404         )) {
2405         $tName = "{$locationType[$locType[1]]}-email";
2406       }
2407       elseif ($locType[0] == 'provider_id') {
2408         $tName = "{$locationType[$locType[1]]}-im";
2409       }
2410       elseif ($locType[0] == 'openid') {
2411         $tName = "{$locationType[$locType[1]]}-openid";
2412       }
2413       else {
2414         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2415       }
2416       $tName = str_replace(' ', '_', $tName);
2417       return [$tName, $fldName];
2418     }
2419     throw new CRM_Core_Exception('Cannot determine location table information');
2420   }
2421
2422   /**
2423    * Given a result dao, extract the values and return that array
2424    *
2425    * @param CRM_Core_DAO $dao
2426    *
2427    * @return array
2428    *   values for this query
2429    */
2430   public function store($dao) {
2431     $value = [];
2432
2433     foreach ($this->_element as $key => $dontCare) {
2434       if (property_exists($dao, $key)) {
2435         if (strpos($key, '-') !== FALSE) {
2436           $values = explode('-', $key);
2437           $lastElement = array_pop($values);
2438           $current = &$value;
2439           $cnt = count($values);
2440           $count = 1;
2441           foreach ($values as $v) {
2442             if (!array_key_exists($v, $current)) {
2443               $current[$v] = [];
2444             }
2445             //bad hack for im_provider
2446             if ($lastElement == 'provider_id') {
2447               if ($count < $cnt) {
2448                 $current = &$current[$v];
2449               }
2450               else {
2451                 $lastElement = "{$v}_{$lastElement}";
2452               }
2453             }
2454             else {
2455               $current = &$current[$v];
2456             }
2457             $count++;
2458           }
2459
2460           $current[$lastElement] = $dao->$key;
2461         }
2462         else {
2463           $value[$key] = $dao->$key;
2464         }
2465       }
2466     }
2467     return $value;
2468   }
2469
2470   /**
2471    * Getter for tables array.
2472    *
2473    * @return array
2474    */
2475   public function tables() {
2476     return $this->_tables;
2477   }
2478
2479   /**
2480    * Sometimes used to create the from clause, but, not reliably, set
2481    * this AND set tables.
2482    *
2483    * It's unclear the intent - there is a 'simpleFrom' clause which
2484    * takes whereTables into account & a fromClause which doesn't.
2485    *
2486    * logic may have eroded?
2487    *
2488    * @return array
2489    */
2490   public function whereTables() {
2491     return $this->_whereTables;
2492   }
2493
2494   /**
2495    * Generate the where clause (used in match contacts and permissions)
2496    *
2497    * @param array $params
2498    * @param array $fields
2499    * @param array $tables
2500    * @param $whereTables
2501    * @param bool $strict
2502    *
2503    * @return string
2504    * @throws \CRM_Core_Exception
2505    */
2506   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2507     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2508       FALSE, $strict
2509     );
2510
2511     $tables = array_merge($query->tables(), $tables);
2512     $whereTables = array_merge($query->whereTables(), $whereTables);
2513
2514     return $query->_whereClause;
2515   }
2516
2517   /**
2518    * Create the from clause.
2519    *
2520    * @param array $tables
2521    *   Tables that need to be included in this from clause. If null,
2522    *   return mimimal from clause (i.e. civicrm_contact).
2523    * @param array $inner
2524    *   Tables that should be inner-joined.
2525    * @param array $right
2526    *   Tables that should be right-joined.
2527    * @param bool $primaryLocation
2528    *   Search on primary location. See note below.
2529    * @param int $mode
2530    *   Determines search mode based on bitwise MODE_* constants.
2531    * @param string|NULL $apiEntity
2532    *   Determines search mode based on entity by string.
2533    *
2534    * The $primaryLocation flag only seems to be used when
2535    * locationType() has been called. This may be a search option
2536    * exposed, or perhaps it's a "search all details" approach which
2537    * predates decoupling of location types and primary fields?
2538    *
2539    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2540    *
2541    * @return string
2542    *   the from clause
2543    */
2544   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2545
2546     $from = ' FROM civicrm_contact contact_a';
2547     if (empty($tables)) {
2548       return $from;
2549     }
2550
2551     if (!empty($tables['civicrm_worldregion'])) {
2552       $tables = array_merge(['civicrm_country' => 1], $tables);
2553     }
2554
2555     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2556         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2557       $tables = array_merge(['civicrm_address' => 1],
2558         $tables
2559       );
2560     }
2561
2562     // add group_contact and group table is subscription history is present
2563     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2564       $tables = array_merge([
2565         'civicrm_group' => 1,
2566         'civicrm_group_contact' => 1,
2567       ],
2568         $tables
2569       );
2570     }
2571
2572     // to handle table dependencies of components
2573     CRM_Core_Component::tableNames($tables);
2574     // to handle table dependencies of hook injected tables
2575     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2576
2577     //format the table list according to the weight
2578     $info = CRM_Core_TableHierarchy::info();
2579
2580     foreach ($tables as $key => $value) {
2581       $k = 99;
2582       if (strpos($key, '-') !== FALSE) {
2583         $keyArray = explode('-', $key);
2584         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2585       }
2586       elseif (strpos($key, '_') !== FALSE) {
2587         $keyArray = explode('_', $key);
2588         if (is_numeric(array_pop($keyArray))) {
2589           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2590         }
2591         else {
2592           $k = CRM_Utils_Array::value($key, $info, 99);
2593         }
2594       }
2595       else {
2596         $k = CRM_Utils_Array::value($key, $info, 99);
2597       }
2598       $tempTable[$k . ".$key"] = $key;
2599     }
2600     ksort($tempTable);
2601     $newTables = [];
2602     foreach ($tempTable as $key) {
2603       $newTables[$key] = $tables[$key];
2604     }
2605
2606     $tables = $newTables;
2607
2608     foreach ($tables as $name => $value) {
2609       if (!$value) {
2610         continue;
2611       }
2612
2613       if (!empty($inner[$name])) {
2614         $side = 'INNER';
2615       }
2616       elseif (!empty($right[$name])) {
2617         $side = 'RIGHT';
2618       }
2619       else {
2620         $side = 'LEFT';
2621       }
2622
2623       if ($value != 1) {
2624         // if there is already a join statement in value, use value itself
2625         if (strpos($value, 'JOIN')) {
2626           $from .= " $value ";
2627         }
2628         else {
2629           $from .= " $side JOIN $name ON ( $value ) ";
2630         }
2631         continue;
2632       }
2633
2634       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2635     }
2636     return $from;
2637   }
2638
2639   /**
2640    * Get join statements for the from clause depending on entity type
2641    *
2642    * @param string $name
2643    * @param int $mode
2644    * @param string $side
2645    * @param string $primaryLocation
2646    * @return string
2647    */
2648   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2649     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2650     switch ($name) {
2651       case 'civicrm_address':
2652         //CRM-14263 further handling of address joins further down...
2653         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2654
2655       case 'civicrm_state_province':
2656         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2657         // but that will have been implied-in by the calling function.
2658         // test cover in testContactIDQuery
2659         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2660
2661       case 'civicrm_country':
2662         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2663         // but that will have been implied-in by the calling function.
2664         // test cover in testContactIDQuery
2665         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2666
2667       case 'civicrm_phone':
2668         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2669
2670       case 'civicrm_email':
2671         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2672
2673       case 'civicrm_im':
2674         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2675
2676       case 'im_provider':
2677         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2678         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2679         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2680         return $from;
2681
2682       case 'civicrm_openid':
2683         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2684
2685       case 'civicrm_worldregion':
2686         // We can be sure from the calling function that country will already be joined in.
2687         // we really don't need world_region - we could use a pseudoconstant for it.
2688         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2689
2690       case 'civicrm_location_type':
2691         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2692
2693       case 'civicrm_group':
2694         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2695
2696       case 'civicrm_group_contact':
2697         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2698
2699       case 'civicrm_group_contact_cache':
2700         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2701
2702       case 'civicrm_activity':
2703       case 'civicrm_activity_tag':
2704       case 'activity_type':
2705       case 'activity_status':
2706       case 'parent_id':
2707       case 'civicrm_activity_contact':
2708       case 'source_contact':
2709       case 'activity_priority':
2710         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2711
2712       case 'civicrm_entity_tag':
2713         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2714         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2715
2716       case 'civicrm_note':
2717         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2718         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2719
2720       case 'civicrm_subscription_history':
2721         $from = " $side JOIN civicrm_subscription_history";
2722         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2723         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2724
2725       case 'civicrm_relationship':
2726         if (self::$_relType == 'reciprocal') {
2727           if (self::$_relationshipTempTable) {
2728             // we have a temptable to join on
2729             $tbl = self::$_relationshipTempTable;
2730             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2731           }
2732           else {
2733             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2734             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2735             return $from;
2736           }
2737         }
2738         elseif (self::$_relType == 'b') {
2739           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2740           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2741         }
2742         else {
2743           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2744           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2745         }
2746
2747       case 'civicrm_log':
2748         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2749         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2750
2751       case 'civicrm_tag':
2752         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2753
2754       case 'civicrm_grant':
2755         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2756
2757       case 'civicrm_website':
2758         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2759
2760       case 'civicrm_campaign':
2761         //Move to default case if not in either mode.
2762         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2763           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2764         }
2765         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2766           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2767         }
2768         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2769           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2770         }
2771
2772       default:
2773         $locationTypeName = '';
2774         if (strpos($name, '-address') != 0) {
2775           $locationTypeName = 'address';
2776         }
2777         elseif (strpos($name, '-phone') != 0) {
2778           $locationTypeName = 'phone';
2779         }
2780         elseif (strpos($name, '-email') != 0) {
2781           $locationTypeName = 'email';
2782         }
2783         elseif (strpos($name, '-im') != 0) {
2784           $locationTypeName = 'im';
2785         }
2786         elseif (strpos($name, '-openid') != 0) {
2787           $locationTypeName = 'openid';
2788         }
2789
2790         if ($locationTypeName) {
2791           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2792           $parts = explode('-', $name);
2793           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2794           foreach ($locationTypes as $locationTypeID => $locationType) {
2795             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2796               $locationID = $locationTypeID;
2797             }
2798           }
2799           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2800         }
2801         else {
2802           $from = CRM_Core_Component::from($name, $mode, $side);
2803         }
2804         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2805
2806         return $from;
2807     }
2808   }
2809
2810   /**
2811    * WHERE / QILL clause for deleted_contacts
2812    *
2813    * @param array $values
2814    */
2815   public function deletedContacts($values) {
2816     list($_, $_, $value, $grouping, $_) = $values;
2817     if ($value) {
2818       // *prepend* to the relevant grouping as this is quite an important factor
2819       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2820     }
2821   }
2822
2823   /**
2824    * Where / qill clause for contact_type
2825    *
2826    * @param $values
2827    *
2828    * @throws \CRM_Core_Exception
2829    */
2830   public function contactType(&$values) {
2831     list($name, $op, $value, $grouping, $wildcard) = $values;
2832
2833     $subTypes = [];
2834     $clause = [];
2835
2836     // account for search builder mapping multiple values
2837     if (!is_array($value)) {
2838       $values = self::parseSearchBuilderString($value, 'String');
2839       if (is_array($values)) {
2840         $value = array_flip($values);
2841       }
2842     }
2843
2844     if (is_array($value)) {
2845       foreach ($value as $k => $v) {
2846         // fix for CRM-771
2847         if ($k) {
2848           $subType = NULL;
2849           $contactType = $k;
2850           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2851             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2852           }
2853
2854           if (!empty($subType)) {
2855             $subTypes[$subType] = 1;
2856           }
2857           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2858         }
2859       }
2860     }
2861     else {
2862       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2863       $contactType = $contactTypeANDSubType[0];
2864       $subType = $contactTypeANDSubType[1] ?? NULL;
2865       if (!empty($subType)) {
2866         $subTypes[$subType] = 1;
2867       }
2868       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2869     }
2870
2871     // fix for CRM-771
2872     if (!empty($clause)) {
2873       $quill = $clause;
2874       if ($op == 'IN' || $op == 'NOT IN') {
2875         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2876       }
2877       else {
2878         $type = array_pop($clause);
2879         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2880       }
2881
2882       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2883
2884       if (!empty($subTypes)) {
2885         $this->includeContactSubTypes($subTypes, $grouping);
2886       }
2887     }
2888   }
2889
2890   /**
2891    * Where / qill clause for contact_sub_type.
2892    *
2893    * @param array $values
2894    */
2895   public function contactSubType(&$values) {
2896     list($name, $op, $value, $grouping, $wildcard) = $values;
2897     $this->includeContactSubTypes($value, $grouping, $op);
2898   }
2899
2900   /**
2901    * @param $value
2902    * @param $grouping
2903    * @param string $op
2904    *
2905    * @throws \CRM_Core_Exception
2906    */
2907   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2908
2909     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2910       $op = key($value);
2911       $value = $value[$op];
2912     }
2913
2914     $clause = [];
2915     $alias = "contact_a.contact_sub_type";
2916     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2917
2918     $op = str_replace('IN', 'LIKE', $op);
2919     $op = str_replace('=', 'LIKE', $op);
2920     $op = str_replace('!', 'NOT ', $op);
2921
2922     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2923       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2924     }
2925     elseif (is_array($value)) {
2926       foreach ($value as $k => $v) {
2927         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2928       }
2929     }
2930     else {
2931       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2932     }
2933
2934     if (!empty($clause)) {
2935       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2936     }
2937     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2938   }
2939
2940   /**
2941    * Where / qill clause for groups.
2942    *
2943    * @param $values
2944    *
2945    * @throws \CRM_Core_Exception
2946    * @throws \Exception
2947    */
2948   public function group($values) {
2949     list($name, $op, $value, $grouping, $wildcard) = $values;
2950
2951     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2952     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2953       $op = key($value);
2954       $value = $value[$op];
2955     }
2956     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2957     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2958     // even one group might equate to multiple when looking at children so IN is simpler.
2959     // @todo - also look at != casting but there are rows below to review.
2960     $opReplacements = [
2961       'IS EMPTY' => 'IS NULL',
2962       'IS NOT EMPTY' => 'IS NOT NULL',
2963       '=' => 'IN',
2964     ];
2965     if (isset($opReplacements[$op])) {
2966       $op = $opReplacements[$op];
2967     }
2968
2969     if (strpos($op, 'NULL')) {
2970       $value = NULL;
2971     }
2972
2973     if (is_array($value) && count($value) > 1) {
2974       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2975         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2976       }
2977       $this->_useDistinct = TRUE;
2978     }
2979
2980     if (isset($value)) {
2981       $value = CRM_Utils_Array::value($op, $value, $value);
2982     }
2983
2984     if ($name === 'group_type') {
2985       $value = array_keys($this->getGroupsFromTypeCriteria($value));
2986     }
2987
2988     $regularGroupIDs = $smartGroupIDs = [];
2989     foreach ((array) $value as $id) {
2990       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
2991         $smartGroupIDs[] = (int) $id;
2992       }
2993       else {
2994         $regularGroupIDs[] = (int) trim($id);
2995       }
2996     }
2997     $hasNonSmartGroups = count($regularGroupIDs);
2998
2999     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3000
3001     $statusJoinClause = $this->getGroupStatusClause($grouping);
3002     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3003     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3004     $groupClause = [];
3005     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3006       // include child groups IDs if any
3007       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3008       foreach ($childGroupIds as $key => $id) {
3009         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3010           $smartGroupIDs[] = $id;
3011           unset($childGroupIds[$key]);
3012         }
3013       }
3014       if (count($childGroupIds)) {
3015         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3016       }
3017
3018       if (empty($regularGroupIDs)) {
3019         if ($isGroupStatusSearch) {
3020           $regularGroupIDs = $smartGroupIDs;
3021         }
3022         // If it is still empty we want a filter that blocks all results.
3023         if (empty($regularGroupIDs)) {
3024           $regularGroupIDs = [0];
3025         }
3026       }
3027
3028       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3029       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3030
3031       // @todo consider just casting != to NOT IN & handling both together.
3032       if ($op === '!=') {
3033         $groupIds = '';
3034         if (!empty($regularGroupIDs)) {
3035           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3036         }
3037         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3038       }
3039       else {
3040         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3041       }
3042       $groupClause[] = "( {$clause} )";
3043
3044       if ($statusJoinClause) {
3045         $joinClause[] = "{$gcTable}.$statusJoinClause";
3046       }
3047       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3048     }
3049
3050     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3051     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3052     // as that info is in the group_contact table.
3053     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3054       $this->_groupUniqueKey = uniqid();
3055       $this->_groupKeys[] = $this->_groupUniqueKey;
3056       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3057       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3058       if (!empty($groupContactCacheClause)) {
3059         if ($isNotOp) {
3060           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3061           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3062           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3063           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3064           if (strpos($op, 'IN') !== FALSE) {
3065             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3066           }
3067           else {
3068             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3069           }
3070         }
3071         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3072       }
3073     }
3074
3075     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3076     if (!empty($groupClause)) {
3077       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3078     }
3079
3080     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3081     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3082     if (strpos($op, 'NULL') === FALSE) {
3083       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3084     }
3085   }
3086
3087   /**
3088    * @return array
3089    */
3090   public function getGroupCacheTableKeys() {
3091     return $this->_groupKeys;
3092   }
3093
3094   /**
3095    * Function translates selection of group type into a list of groups.
3096    * @param $value
3097    *
3098    * @return array
3099    */
3100   public function getGroupsFromTypeCriteria($value) {
3101     $groupIds = [];
3102     foreach ((array) $value as $groupTypeValue) {
3103       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3104       $groupIds = ($groupIds + $groupList);
3105     }
3106     return $groupIds;
3107   }
3108
3109   /**
3110    * Prime smart group cache for smart groups in the search, and join
3111    * civicrm_group_contact_cache table into the query.
3112    *
3113    * @param array $groups IDs of groups specified in search criteria.
3114    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3115    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3116    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3117    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3118    *
3119    * @return string WHERE clause component for smart group criteria.
3120    * @throws \CRM_Core_Exception
3121    */
3122   public function addGroupContactCache($groups, $tableAlias, $joinTable = "contact_a", $op, $joinColumn = 'id') {
3123     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3124     $groupsIds = $groups;
3125
3126     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3127     if (!empty($operator[$op]) && is_array($groups)) {
3128       $op = $operator[$op];
3129     }
3130     if (!$isNullOp && !$groups) {
3131       return NULL;
3132     }
3133     elseif (strpos($op, 'IN') !== FALSE) {
3134       $groups = [$op => $groups];
3135     }
3136     elseif (is_array($groups) && count($groups)) {
3137       $groups = ['IN' => $groups];
3138     }
3139
3140     // Find all the groups that are part of a saved search.
3141     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3142     $sql = "
3143 SELECT id, cache_date, saved_search_id, children
3144 FROM   civicrm_group
3145 WHERE  $smartGroupClause
3146   AND  ( saved_search_id != 0
3147    OR    saved_search_id IS NOT NULL
3148    OR    children IS NOT NULL )
3149 ";
3150
3151     $group = CRM_Core_DAO::executeQuery($sql);
3152
3153     while ($group->fetch()) {
3154       $this->_useDistinct = TRUE;
3155       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3156         CRM_Contact_BAO_GroupContactCache::load($group);
3157       }
3158     }
3159     if ($group->N == 0 && $op != 'NOT IN') {
3160       return NULL;
3161     }
3162
3163     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3164
3165     if ($op == 'NOT IN') {
3166       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3167     }
3168     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3169   }
3170
3171   /**
3172    * Where / qill clause for cms users
3173    *
3174    * @param $values
3175    */
3176   public function ufUser(&$values) {
3177     list($name, $op, $value, $grouping, $wildcard) = $values;
3178
3179     if ($value == 1) {
3180       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3181
3182       $this->_qill[$grouping][] = ts('CMS User');
3183     }
3184     elseif ($value == 0) {
3185       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3186
3187       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3188       $this->_qill[$grouping][] = ts('Not a CMS User');
3189     }
3190   }
3191
3192   /**
3193    * All tag search specific.
3194    *
3195    * @param array $values
3196    *
3197    * @throws \CRM_Core_Exception
3198    */
3199   public function tagSearch(&$values) {
3200     list($name, $op, $value, $grouping, $wildcard) = $values;
3201
3202     $op = "LIKE";
3203     $value = "%{$value}%";
3204     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3205
3206     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3207     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3208
3209     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3210     $tTable = "`civicrm_tag-" . uniqid() . "`";
3211
3212     if ($useAllTagTypes[2]) {
3213       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3214         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3215             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3216
3217       // search tag in cases
3218       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3219       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3220       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3221         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3222             LEFT JOIN civicrm_case
3223             ON (civicrm_case_contact.case_id = civicrm_case.id
3224                 AND civicrm_case.is_deleted = 0 )
3225             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3226             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3227       // search tag in activities
3228       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3229       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3230       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3231       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3232
3233       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3234         = " LEFT JOIN civicrm_activity_contact
3235             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3236             LEFT JOIN civicrm_activity
3237             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3238             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3239             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3240             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3241
3242       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3243       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3244     }
3245     else {
3246       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3247       $tTable = "`civicrm_tag-" . uniqid() . "`";
3248       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3249       {$etTable}.entity_table = 'civicrm_contact' )
3250                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3251
3252       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3253       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3254     }
3255   }
3256
3257   /**
3258    * Where / qill clause for tag.
3259    *
3260    * @param array $values
3261    *
3262    * @throws \CRM_Core_Exception
3263    */
3264   public function tag(&$values) {
3265     list($name, $op, $value, $grouping, $wildcard) = $values;
3266
3267     // API/Search Builder format array(operator => array(values))
3268     if (is_array($value)) {
3269       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3270         $op = key($value);
3271         $value = $value[$op];
3272       }
3273       if (count($value) > 1) {
3274         $this->_useDistinct = TRUE;
3275       }
3276     }
3277
3278     if (strpos($op, 'NULL') || strpos($op, 'EMPTY')) {
3279       $value = NULL;
3280     }
3281
3282     $tagTree = CRM_Core_BAO_Tag::getChildTags();
3283     foreach ((array) $value as $tagID) {
3284       if (!empty($tagTree[$tagID])) {
3285         $value = array_unique(array_merge($value, $tagTree[$tagID]));
3286       }
3287     }
3288
3289     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3290
3291     // implode array, then remove all spaces
3292     $value = str_replace(' ', '', implode(',', (array) $value));
3293     if (!empty($value)) {
3294       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3295     }
3296
3297     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3298     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3299
3300     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3301
3302     if (!empty($useAllTagTypes[2])) {
3303       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3304         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3305
3306       // search tag in cases
3307       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3308       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3309       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3310
3311       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3312         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3313             LEFT JOIN civicrm_case
3314             ON (civicrm_case_contact.case_id = civicrm_case.id
3315                 AND civicrm_case.is_deleted = 0 )
3316             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3317       // search tag in activities
3318       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3319       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3320         = " LEFT JOIN civicrm_activity_contact
3321             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3322             LEFT JOIN civicrm_activity
3323             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3324             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3325             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3326
3327       // CRM-10338
3328       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3329         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3330       }
3331       else {
3332         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3333       }
3334     }
3335     else {
3336       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3337         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3338
3339       // CRM-10338
3340       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3341         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3342         $op = str_replace('EMPTY', 'NULL', $op);
3343         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3344       }
3345       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3346       elseif ($op == '!=') {
3347         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3348       }
3349       elseif ($op == '=' || strstr($op, 'IN')) {
3350         $op = ($op == '=') ? 'IN' : $op;
3351         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3352       }
3353     }
3354     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3355   }
3356
3357   /**
3358    * Where/qill clause for notes
3359    *
3360    * @param array $values
3361    *
3362    * @throws \CRM_Core_Exception
3363    */
3364   public function notes(&$values) {
3365     list($name, $op, $value, $grouping, $wildcard) = $values;
3366
3367     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3368     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3369     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3370
3371     $this->_useDistinct = TRUE;
3372
3373     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3374       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3375
3376     $n = trim($value);
3377     $value = CRM_Core_DAO::escapeString($n);
3378     if ($wildcard) {
3379       if (strpos($value, '%') === FALSE) {
3380         $value = "%$value%";
3381       }
3382       $op = 'LIKE';
3383     }
3384     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3385       $value = NULL;
3386     }
3387
3388     $label = NULL;
3389     $clauses = [];
3390     if ($noteOption % 2 == 0) {
3391       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3392       $label = ts('Note: Body Only');
3393     }
3394     if ($noteOption % 3 == 0) {
3395       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3396       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3397     }
3398     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3399     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3400     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3401   }
3402
3403   /**
3404    * @param string $name
3405    * @param $op
3406    * @param $grouping
3407    *
3408    * @return bool
3409    */
3410   public function nameNullOrEmptyOp($name, $op, $grouping) {
3411     switch ($op) {
3412       case 'IS NULL':
3413       case 'IS NOT NULL':
3414         $this->_where[$grouping][] = "contact_a.$name $op";
3415         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3416         return TRUE;
3417
3418       case 'IS EMPTY':
3419         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3420         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3421         return TRUE;
3422
3423       case 'IS NOT EMPTY':
3424         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3425         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3426         return TRUE;
3427
3428       default:
3429         return FALSE;
3430     }
3431   }
3432
3433   /**
3434    * Where / qill clause for sort_name
3435    *
3436    * @param array $values
3437    */
3438   public function sortName(&$values) {
3439     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3440
3441     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3442     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3443       return;
3444     }
3445
3446     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3447
3448     if (!strlen($value)) {
3449       return;
3450     }
3451
3452     $config = CRM_Core_Config::singleton();
3453
3454     $sub = [];
3455
3456     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3457     $subGlue = ' OR ';
3458
3459     $firstChar = substr($value, 0, 1);
3460     $lastChar = substr($value, -1, 1);
3461     $quotes = ["'", '"'];
3462     // If string is quoted, strip quotes and otherwise don't alter it
3463     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3464       $value = trim($value, implode('', $quotes));
3465     }
3466     // Replace spaces with wildcards for a LIKE operation
3467     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3468     // Also need to check if there is space in between sort name.
3469     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') === TRUE) {
3470       $value = str_replace(' ', '%', $value);
3471     }
3472     $value = CRM_Core_DAO::escapeString(trim($value));
3473     if (strlen($value)) {
3474       $fieldsub = [];
3475       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3476       if ($fieldName == 'sort_name') {
3477         $wc = "contact_a.sort_name";
3478       }
3479       else {
3480         $wc = "contact_a.display_name";
3481       }
3482       $fieldsub[] = " ( $wc $op $value )";
3483       if ($config->includeNickNameInName) {
3484         $wc = "contact_a.nick_name";
3485         $fieldsub[] = " ( $wc $op $value )";
3486       }
3487       if ($config->includeEmailInName) {
3488         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3489       }
3490       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3491     }
3492
3493     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3494
3495     $this->_where[$grouping][] = $sub;
3496     if ($config->includeEmailInName) {
3497       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3498       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3499     }
3500     else {
3501       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3502     }
3503   }
3504
3505   /**
3506    * Where/qill clause for greeting fields.
3507    *
3508    * @param array $values
3509    *
3510    * @throws \CRM_Core_Exception
3511    */
3512   public function greetings(&$values) {
3513     list($name, $op, $value, $grouping, $wildcard) = $values;
3514     $name .= '_display';
3515
3516     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3517     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3518     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3519   }
3520
3521   /**
3522    * Where / qill clause for email
3523    *
3524    * @param array $values
3525    * @param string $isForcePrimaryOnly
3526    *
3527    * @throws \CRM_Core_Exception
3528    */
3529   protected function email(&$values, $isForcePrimaryOnly) {
3530     list($name, $op, $value, $grouping, $wildcard) = $values;
3531     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3532
3533     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3534     // so following code is use build whereClause for contact's primart email id
3535     if (!empty($isForcePrimaryOnly)) {
3536       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3537     }
3538     // @todo - this should come from the $this->_fields array
3539     $dbName = $name === 'email_id' ? 'id' : $name;
3540
3541     if (is_array($value) || $name === 'email_id') {
3542       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3543       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3544       return;
3545     }
3546
3547     // Is this ever hit now? Ideally ensure always an array & handle above.
3548     $n = trim($value);
3549     if ($n) {
3550       if (substr($n, 0, 1) == '"' &&
3551         substr($n, -1, 1) == '"'
3552       ) {
3553         $n = substr($n, 1, -1);
3554         $value = CRM_Core_DAO::escapeString($n);
3555         $op = '=';
3556       }
3557       else {
3558         $value = self::getWildCardedValue($wildcard, $op, $n);
3559       }
3560       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3561       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3562     }
3563     else {
3564       $this->_qill[$grouping][] = ts('Email') . " $op ";
3565       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3566     }
3567   }
3568
3569   /**
3570    * Where / qill clause for phone number
3571    *
3572    * @param array $values
3573    *
3574    * @throws \CRM_Core_Exception
3575    */
3576   public function phone_numeric(&$values) {
3577     list($name, $op, $value, $grouping, $wildcard) = $values;
3578     // Strip non-numeric characters; allow wildcards
3579     $number = preg_replace('/[^\d%]/', '', $value);
3580     if ($number) {
3581       if (strpos($number, '%') === FALSE) {
3582         $number = "%$number%";
3583       }
3584
3585       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3586       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3587       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3588     }
3589   }
3590
3591   /**
3592    * Where / qill clause for phone type/location
3593    *
3594    * @param array $values
3595    *
3596    * @throws \CRM_Core_Exception
3597    */
3598   public function phone_option_group($values) {
3599     list($name, $op, $value, $grouping, $wildcard) = $values;
3600     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3601     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3602     $optionName = $options[$value];
3603     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3604     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3605     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3606   }
3607
3608   /**
3609    * Where / qill clause for street_address.
3610    *
3611    * @param array $values
3612    *
3613    * @throws \CRM_Core_Exception
3614    */
3615   public function street_address(&$values) {
3616     list($name, $op, $value, $grouping) = $values;
3617
3618     if (!$op) {
3619       $op = 'LIKE';
3620     }
3621
3622     $n = trim($value);
3623
3624     if ($n) {
3625       if (strpos($value, '%') === FALSE) {
3626         // only add wild card if not there
3627         $value = "%{$value}%";
3628       }
3629       $op = 'LIKE';
3630       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3631       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3632     }
3633     else {
3634       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3635       $this->_qill[$grouping][] = ts('Street') . " $op ";
3636     }
3637
3638     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3639   }
3640
3641   /**
3642    * Where / qill clause for street_unit.
3643    *
3644    * @param array $values
3645    *
3646    * @throws \CRM_Core_Exception
3647    */
3648   public function street_number(&$values) {
3649     list($name, $op, $value, $grouping, $wildcard) = $values;
3650
3651     if (!$op) {
3652       $op = '=';
3653     }
3654
3655     $n = trim($value);
3656
3657     if (strtolower($n) == 'odd') {
3658       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3659       $this->_qill[$grouping][] = ts('Street Number is odd');
3660     }
3661     elseif (strtolower($n) == 'even') {
3662       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3663       $this->_qill[$grouping][] = ts('Street Number is even');
3664     }
3665     else {
3666       $value = $n;
3667       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3668       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3669     }
3670
3671     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3672   }
3673
3674   /**
3675    * Where / qill clause for sorting by character.
3676    *
3677    * @param array $values
3678    */
3679   public function sortByCharacter(&$values) {
3680     list($name, $op, $value, $grouping, $wildcard) = $values;
3681
3682     $name = trim($value);
3683     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3684     $this->_where[$grouping][] = $cond;
3685     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3686   }
3687
3688   /**
3689    * Where / qill clause for including contact ids.
3690    */
3691   public function includeContactIDs() {
3692     if (!$this->_includeContactIds || empty($this->_params)) {
3693       return;
3694     }
3695
3696     $contactIds = [];
3697     foreach ($this->_params as $id => $values) {
3698       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3699         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3700       }
3701     }
3702     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3703     if (!empty($contactIds)) {
3704       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3705     }
3706   }
3707
3708   /**
3709    * Where / qill clause for postal code.
3710    *
3711    * @param array $values
3712    *
3713    * @throws \CRM_Core_Exception
3714    */
3715   public function postalCode(&$values) {
3716     // skip if the fields dont have anything to do with postal_code
3717     if (empty($this->_fields['postal_code'])) {
3718       return;
3719     }
3720
3721     list($name, $op, $value, $grouping, $wildcard) = $values;
3722
3723     // Handle numeric postal code range searches properly by casting the column as numeric
3724     if (is_numeric($value)) {
3725       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3726       $val = CRM_Utils_Type::escape($value, 'Integer');
3727     }
3728     else {
3729       $field = 'civicrm_address.postal_code';
3730       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3731       if (!is_array($value)) {
3732         $val = CRM_Utils_Type::escape($value, 'String');
3733       }
3734       else {
3735         // Do we need to escape values here? I would expect buildClause does.
3736         $val = $value;
3737       }
3738     }
3739
3740     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3741
3742     if ($name == 'postal_code') {
3743       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3744       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3745     }
3746     elseif ($name == 'postal_code_low') {
3747       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3748       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3749     }
3750     elseif ($name == 'postal_code_high') {
3751       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3752       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3753     }
3754   }
3755
3756   /**
3757    * Where / qill clause for location type.
3758    *
3759    * @param array $values
3760    * @param null $status
3761    *
3762    * @return string
3763    */
3764   public function locationType(&$values, $status = NULL) {
3765     list($name, $op, $value, $grouping, $wildcard) = $values;
3766
3767     if (is_array($value)) {
3768       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3769       $this->_tables['civicrm_address'] = 1;
3770       $this->_whereTables['civicrm_address'] = 1;
3771
3772       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3773       $names = [];
3774       foreach ($value as $id) {
3775         $names[] = $locationType[$id];
3776       }
3777
3778       $this->_primaryLocation = FALSE;
3779
3780       if (!$status) {
3781         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3782       }
3783       else {
3784         return implode(' ' . ts('or') . ' ', $names);
3785       }
3786     }
3787   }
3788
3789   /**
3790    * @param $values
3791    * @param bool $fromStateProvince
3792    *
3793    * @return array|NULL
3794    * @throws \CRM_Core_Exception
3795    */
3796   public function country(&$values, $fromStateProvince = TRUE) {
3797     list($name, $op, $value, $grouping, $wildcard) = $values;
3798
3799     if (!$fromStateProvince) {
3800       $stateValues = $this->getWhereValues('state_province', $grouping);
3801       if (!empty($stateValues)) {
3802         // return back to caller if there are state province values
3803         // since that handles this case
3804         return NULL;
3805       }
3806     }
3807
3808     $countryClause = $countryQill = NULL;
3809     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3810       $this->_tables['civicrm_address'] = 1;
3811       $this->_whereTables['civicrm_address'] = 1;
3812
3813       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3814       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3815       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3816
3817       if (!$fromStateProvince) {
3818         $this->_where[$grouping][] = $countryClause;
3819         $this->_qill[$grouping][] = $countryQill;
3820       }
3821     }
3822
3823     if ($fromStateProvince) {
3824       if (!empty($countryClause)) {
3825         return [
3826           $countryClause,
3827           " ...AND... " . $countryQill,
3828         ];
3829       }
3830       else {
3831         return [NULL, NULL];
3832       }
3833     }
3834   }
3835
3836   /**
3837    * Where / qill clause for county (if present).
3838    *
3839    * @param array $values
3840    * @param null $status
3841    *
3842    * @return string
3843    */
3844   public function county(&$values, $status = NULL) {
3845     list($name, $op, $value, $grouping, $wildcard) = $values;
3846
3847     if (!is_array($value)) {
3848       // force the county to be an array
3849       $value = [$value];
3850     }
3851
3852     // check if the values are ids OR names of the counties
3853     $inputFormat = 'id';
3854     foreach ($value as $v) {
3855       if (!is_numeric($v)) {
3856         $inputFormat = 'name';
3857         break;
3858       }
3859     }
3860     $names = [];
3861     if ($op == '=') {
3862       $op = 'IN';
3863     }
3864     elseif ($op == '!=') {
3865       $op = 'NOT IN';
3866     }
3867     else {
3868       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3869       $op = str_replace('EMPTY', 'NULL', $op);
3870     }
3871
3872     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3873       $clause = "civicrm_address.county_id $op";
3874     }
3875     elseif ($inputFormat == 'id') {
3876       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3877
3878       $county = CRM_Core_PseudoConstant::county();
3879       foreach ($value as $id) {
3880         $names[] = $county[$id] ?? NULL;
3881       }
3882     }
3883     else {
3884       $inputClause = [];
3885       $county = CRM_Core_PseudoConstant::county();
3886       foreach ($value as $name) {
3887         $name = trim($name);
3888         $inputClause[] = CRM_Utils_Array::key($name, $county);
3889       }
3890       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3891       $names = $value;
3892     }
3893     $this->_tables['civicrm_address'] = 1;
3894     $this->_whereTables['civicrm_address'] = 1;
3895
3896     $this->_where[$grouping][] = $clause;
3897     if (!$status) {
3898       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3899     }
3900     else {
3901       return implode(' ' . ts('or') . ' ', $names);
3902     }
3903   }
3904
3905   /**
3906    * Where / qill clause for state/province AND country (if present).
3907    *
3908    * @param array $values
3909    * @param null $status
3910    *
3911    * @return string
3912    * @throws \CRM_Core_Exception
3913    */
3914   public function stateProvince(&$values, $status = NULL) {
3915     list($name, $op, $value, $grouping, $wildcard) = $values;
3916
3917     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3918     $this->_tables['civicrm_address'] = 1;
3919     $this->_whereTables['civicrm_address'] = 1;
3920
3921     $countryValues = $this->getWhereValues('country', $grouping);
3922     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3923     if ($countryClause) {
3924       $clause = "( $stateClause AND $countryClause )";
3925     }
3926     else {
3927       $clause = $stateClause;
3928     }
3929
3930     $this->_where[$grouping][] = $clause;
3931     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3932     if (!$status) {
3933       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3934     }
3935     else {
3936       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3937     }
3938   }
3939
3940   /**
3941    * Where / qill clause for change log.
3942    *
3943    * @param array $values
3944    */
3945   public function changeLog(&$values) {
3946     list($name, $op, $value, $grouping, $wildcard) = $values;
3947
3948     $targetName = $this->getWhereValues('changed_by', $grouping);
3949     if (!$targetName) {
3950       return;
3951     }
3952
3953     $name = trim($targetName[2]);
3954     $name = CRM_Core_DAO::escapeString($name);
3955     $name = $targetName[4] ? "%$name%" : $name;
3956     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3957     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3958     $fieldTitle = ts('Altered By');
3959
3960     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3961     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3962       1 => $fieldTitle,
3963       2 => $qillop,
3964       3 => $qillVal,
3965     ]);
3966   }
3967
3968   /**
3969    * @param $values
3970    *
3971    * @throws \CRM_Core_Exception
3972    */
3973   public function modifiedDates($values) {
3974     $this->_useDistinct = TRUE;
3975
3976     // CRM-11281, default to added date if not set
3977     $fieldTitle = ts('Added Date');
3978     $fieldName = 'created_date';
3979     foreach (array_keys($this->_params) as $id) {
3980       if ($this->_params[$id][0] == 'log_date') {
3981         if ($this->_params[$id][2] == 2) {
3982           $fieldTitle = ts('Modified Date');
3983           $fieldName = 'modified_date';
3984         }
3985       }
3986     }
3987
3988     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
3989
3990     self::$_openedPanes[ts('Change Log')] = TRUE;
3991   }
3992
3993   /**
3994    * @param $values
3995    *
3996    * @throws \CRM_Core_Exception
3997    */
3998   public function demographics(&$values) {
3999     list($name, $op, $value, $grouping, $wildcard) = $values;
4000
4001     if (($name == 'age_low') || ($name == 'age_high')) {
4002       $this->ageRangeQueryBuilder($values,
4003         'contact_a', 'age', 'birth_date', ts('Age')
4004       );
4005     }
4006     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4007
4008       $this->dateQueryBuilder($values,
4009         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4010       );
4011     }
4012     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4013
4014       $this->dateQueryBuilder($values,
4015         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4016       );
4017     }
4018
4019     self::$_openedPanes[ts('Demographics')] = TRUE;
4020   }
4021
4022   /**
4023    * @param $values
4024    */
4025   public function privacy(&$values) {
4026     list($name, $op, $value, $grouping) = $values;
4027     if (is_array($value)) {
4028       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4029         $op = key($value);
4030         $value = $value[$op];
4031       }
4032     }
4033     $field = $this->_fields[$name] ?? NULL;
4034     CRM_Utils_Type::validate($value, 'Integer');
4035     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4036     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4037     $title = $field ? $field['title'] : $name;
4038     $this->_qill[$grouping][] = "$title $op $value";
4039   }
4040
4041   /**
4042    * @param $values
4043    */
4044   public function privacyOptions($values) {
4045     list($name, $op, $value, $grouping, $wildcard) = $values;
4046
4047     if (empty($value) || !is_array($value)) {
4048       return;
4049     }
4050
4051     // get the operator and toggle values
4052     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4053     $operator = 'OR';
4054     if ($opValues &&
4055       strtolower($opValues[2] == 'AND')
4056     ) {
4057       // @todo this line is logially unreachable
4058       $operator = 'AND';
4059     }
4060
4061     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4062     $compareOP = '!';
4063     if ($toggleValues &&
4064       $toggleValues[2] == 2
4065     ) {
4066       $compareOP = '';
4067     }
4068
4069     $clauses = [];
4070     $qill = [];
4071     foreach ($value as $dontCare => $pOption) {
4072       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4073       $field = $this->_fields[$pOption] ?? NULL;
4074       $title = $field ? $field['title'] : $pOption;
4075       $qill[] = " $title = 1 ";
4076     }
4077
4078     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4079     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4080   }
4081
4082   /**
4083    * @param $values
4084    *
4085    * @throws \CRM_Core_Exception
4086    */
4087   public function preferredCommunication(&$values) {
4088     list($name, $op, $value, $grouping, $wildcard) = $values;
4089
4090     if (!is_array($value)) {
4091       $value = str_replace(['(', ')'], '', explode(",", $value));
4092     }
4093     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4094       $op = key($value);
4095       $value = $value[$op];
4096     }
4097     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4098
4099     if (self::caseImportant($op)) {
4100       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4101       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4102       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4103     }
4104
4105     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4106     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4107   }
4108
4109   /**
4110    * Where / qill clause for relationship.
4111    *
4112    * @param array $values
4113    */
4114   public function relationship(&$values) {
4115     list($name, $op, $value, $grouping, $wildcard) = $values;
4116     if ($this->_relationshipValuesAdded) {
4117       return;
4118     }
4119     // also get values array for relation_target_name
4120     // for relationship search we always do wildcard
4121     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4122     $description = $this->getWhereValues('relation_description', $grouping);
4123     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4124     $relStatus = $this->getWhereValues('relation_status', $grouping);
4125     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4126
4127     $nameClause = $name = NULL;
4128     if ($targetName) {
4129       $name = trim($targetName[2]);
4130       if (substr($name, 0, 1) == '"' &&
4131         substr($name, -1, 1) == '"'
4132       ) {
4133         $name = substr($name, 1, -1);
4134         $name = CRM_Core_DAO::escapeString($name);
4135         $nameClause = "= '$name'";
4136       }
4137       else {
4138         $name = CRM_Core_DAO::escapeString($name);
4139         $nameClause = "LIKE '%{$name}%'";
4140       }
4141     }
4142
4143     $relTypes = $relTypesIds = [];
4144     if (!empty($relationType)) {
4145       $relationType[2] = (array) $relationType[2];
4146       foreach ($relationType[2] as $relType) {
4147         $rel = explode('_', $relType);
4148         self::$_relType = $rel[1];
4149         $params = ['id' => $rel[0]];
4150         $typeValues = [];
4151         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4152         if (!empty($rTypeValue)) {
4153           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4154             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4155             self::$_relType = 'reciprocal';
4156           }
4157           $relTypesIds[] = $rel[0];
4158           $relTypes[] = $relType;
4159         }
4160       }
4161     }
4162
4163     // if we are creating a temp table we build our own where for the relationship table
4164     $relationshipTempTable = NULL;
4165     if (self::$_relType == 'reciprocal') {
4166       $where = [];
4167       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4168         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4169         ->getName();
4170       if ($nameClause) {
4171         $where[$grouping][] = " sort_name $nameClause ";
4172       }
4173       $groupJoinTable = "civicrm_relationship";
4174       $groupJoinColumn = "contact_id_alt";
4175     }
4176     else {
4177       $where = &$this->_where;
4178       if ($nameClause) {
4179         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4180       }
4181       $groupJoinTable = "contact_b";
4182       $groupJoinColumn = "id";
4183     }
4184     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4185     if ($nameClause || !$targetGroup) {
4186       if (!empty($relationType)) {
4187         $relQill = '';
4188         foreach ($relTypes as $rel) {
4189           if (!empty($relQill)) {
4190             $relQill .= ' OR ';
4191           }
4192           $relQill .= $allRelationshipType[$rel];
4193         }
4194         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4195       }
4196       elseif ($name) {
4197         $this->_qill[$grouping][] = $name;
4198       }
4199     }
4200
4201     //check to see if the target contact is in specified group
4202     if ($targetGroup) {
4203       //add contacts from static groups
4204       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4205         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4206       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4207         implode(",", $targetGroup[2]) . ") ) ";
4208
4209       //add contacts from saved searches
4210       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4211
4212       //set the group where clause
4213       if ($ssWhere) {
4214         $groupWhere[] = "( " . $ssWhere . " )";
4215       }
4216       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4217
4218       //Get the names of the target groups for the qill
4219       $groupNames = CRM_Core_PseudoConstant::group();
4220       $qillNames = [];
4221       foreach ($targetGroup[2] as $groupId) {
4222         if (array_key_exists($groupId, $groupNames)) {
4223           $qillNames[] = $groupNames[$groupId];
4224         }
4225       }
4226       if (!empty($relationType)) {
4227         $relQill = '';
4228         foreach ($relTypes as $rel) {
4229           if (!empty($relQill)) {
4230             $relQill .= ' OR ';
4231           }
4232           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4233         }
4234         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4235       }
4236       else {
4237         $this->_qill[$grouping][] = implode(", ", $qillNames);
4238       }
4239     }
4240
4241     // Description
4242     if (!empty($description[2]) && trim($description[2])) {
4243       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4244       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4245       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4246     }
4247
4248     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4249     $today = date('Ymd');
4250     //check for active, inactive and all relation status
4251     if (empty($relStatus[2])) {
4252       $where[$grouping][] = "(
4253 civicrm_relationship.is_active = 1 AND
4254 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4255 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4256 )";
4257       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4258     }
4259     elseif ($relStatus[2] == 1) {
4260       $where[$grouping][] = "(
4261 civicrm_relationship.is_active = 0 OR
4262 civicrm_relationship.end_date < {$today} OR
4263 civicrm_relationship.start_date > {$today}
4264 )";
4265       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4266     }
4267
4268     $onlyDeleted = 0;
4269     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4270       $onlyDeleted = 1;
4271     }
4272     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4273
4274     $this->addRelationshipPermissionClauses($grouping, $where);
4275     $this->addRelationshipDateClauses($grouping, $where);
4276     $this->addRelationshipActivePeriodClauses($grouping, $where);
4277     if (!empty($relTypes)) {
4278       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4279     }
4280     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4281     $this->_useDistinct = TRUE;
4282     $this->_relationshipValuesAdded = TRUE;
4283     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4284     // join on that,
4285     if ($relationshipTempTable) {
4286       $whereClause = '';
4287       if (!empty($where[$grouping])) {
4288         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4289         $whereClause = str_replace('contact_b', 'c', $whereClause);
4290       }
4291       $sql = "
4292         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4293           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4294             FROM civicrm_relationship
4295             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4296             $whereClause )
4297           UNION
4298             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4299             FROM civicrm_relationship
4300             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4301             $whereClause )
4302       ";
4303       CRM_Core_DAO::executeQuery($sql);
4304     }
4305   }
4306
4307   /**
4308    * Add relationship permission criteria to where clause.
4309    *
4310    * @param string $grouping
4311    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4312    *   Not the main query.
4313    */
4314   public function addRelationshipPermissionClauses($grouping, &$where) {
4315     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4316     if ($relPermission) {
4317       if (!is_array($relPermission[2])) {
4318         // this form value was scalar in previous versions of Civi
4319         $relPermission[2] = [$relPermission[2]];
4320       }
4321       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4322
4323       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4324
4325       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4326       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4327     }
4328   }
4329
4330   /**
4331    * Add start & end date criteria in
4332    * @param string $grouping
4333    * @param array $where
4334    *   = array to add where clauses to, in case you are generating a temp table.
4335    * not the main query.
4336    */
4337   public function addRelationshipDateClauses($grouping, &$where) {
4338     foreach (['start_date', 'end_date'] as $dateField) {
4339       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4340       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4341       if (!empty($dateValueLow)) {
4342         $date = date('Ymd', strtotime($dateValueLow[2]));
4343         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4344         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4345       }
4346       if (!empty($dateValueHigh)) {
4347         $date = date('Ymd', strtotime($dateValueHigh[2]));
4348         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4349         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4350       }
4351     }
4352   }
4353
4354   /**
4355    * Add start & end active period criteria in
4356    * @param string $grouping
4357    * @param array $where
4358    *   = array to add where clauses to, in case you are generating a temp table.
4359    * not the main query.
4360    */
4361   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4362     $dateValues = [];
4363     $dateField = 'active_period_date';
4364
4365     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4366     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4367     $dateValueLowFormated = $dateValueHighFormated = NULL;
4368     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4369       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4370       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4371       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4372     }
4373     elseif (!empty($dateValueLow)) {
4374       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4375       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4376     }
4377     elseif (!empty($dateValueHigh)) {
4378       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4379       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4380     }
4381
4382     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4383       $where[$grouping][] = $activePeriodClauses;
4384     }
4385   }
4386
4387   /**
4388    * Get start & end active period criteria
4389    *
4390    * @param $from
4391    * @param $to
4392    * @param $forceTableName
4393    *
4394    * @return string
4395    */
4396   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4397     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4398     if (!is_null($from) && !is_null($to)) {
4399       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4400                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4401                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4402                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4403                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4404                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4405     }
4406     elseif (!is_null($from)) {
4407       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4408                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4409                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4410     }
4411     elseif (!is_null($to)) {
4412       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4413                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4414                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4415     }
4416   }
4417
4418   /**
4419    * Default set of return properties.
4420    *
4421    * @param int $mode
4422    *
4423    * @return array
4424    *   derault return properties
4425    */
4426   public static function &defaultReturnProperties($mode = 1) {
4427     if (!isset(self::$_defaultReturnProperties)) {
4428       self::$_defaultReturnProperties = [];
4429     }
4430
4431     if (!isset(self::$_defaultReturnProperties[$mode])) {
4432       // add activity return properties
4433       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4434         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4435       }
4436       else {
4437         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4438       }
4439
4440       if (empty(self::$_defaultReturnProperties[$mode])) {
4441         self::$_defaultReturnProperties[$mode] = [
4442           'home_URL' => 1,
4443           'image_URL' => 1,
4444           'legal_identifier' => 1,
4445           'external_identifier' => 1,
4446           'contact_type' => 1,
4447           'contact_sub_type' => 1,
4448           'sort_name' => 1,
4449           'display_name' => 1,
4450           'preferred_mail_format' => 1,
4451           'nick_name' => 1,
4452           'first_name' => 1,
4453           'middle_name' => 1,
4454           'last_name' => 1,
4455           'prefix_id' => 1,
4456           'suffix_id' => 1,
4457           'formal_title' => 1,
4458           'communication_style_id' => 1,
4459           'birth_date' => 1,
4460           'gender_id' => 1,
4461           'street_address' => 1,
4462           'supplemental_address_1' => 1,
4463           'supplemental_address_2' => 1,
4464           'supplemental_address_3' => 1,
4465           'city' => 1,
4466           'postal_code' => 1,
4467           'postal_code_suffix' => 1,
4468           'state_province' => 1,
4469           'country' => 1,
4470           'world_region' => 1,
4471           'geo_code_1' => 1,
4472           'geo_code_2' => 1,
4473           'email' => 1,
4474           'on_hold' => 1,
4475           'phone' => 1,
4476           'im' => 1,
4477           'household_name' => 1,
4478           'organization_name' => 1,
4479           'deceased_date' => 1,
4480           'is_deceased' => 1,
4481           'job_title' => 1,
4482           'legal_name' => 1,
4483           'sic_code' => 1,
4484           'current_employer' => 1,
4485           // FIXME: should we use defaultHierReturnProperties() for the below?
4486           'do_not_email' => 1,
4487           'do_not_mail' => 1,
4488           'do_not_sms' => 1,
4489           'do_not_phone' => 1,
4490           'do_not_trade' => 1,
4491           'is_opt_out' => 1,
4492           'contact_is_deleted' => 1,
4493           'preferred_communication_method' => 1,
4494           'preferred_language' => 1,
4495         ];
4496       }
4497     }
4498     return self::$_defaultReturnProperties[$mode];
4499   }
4500
4501   /**
4502    * Get primary condition for a sql clause.
4503    *
4504    * @param int $value
4505    *
4506    * @return string|NULL
4507    */
4508   public static function getPrimaryCondition($value) {
4509     if (is_numeric($value)) {
4510       $value = (int ) $value;
4511       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4512     }
4513     return NULL;
4514   }
4515
4516   /**
4517    * Wrapper for a simple search query.
4518    *
4519    * @param array $params
4520    * @param array $returnProperties
4521    * @param bool $count
4522    *
4523    * @return string
4524    * @throws \CRM_Core_Exception
4525    */
4526   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4527     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4528     list($select, $from, $where, $having) = $query->query();
4529     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4530
4531     $query = "$select $from $where $groupBy $having";
4532     return $query;
4533   }
4534
4535   /**
4536    * These are stub comments as this function needs more explanation - particularly in terms of how it
4537    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4538    *
4539    * This function was originally written as a wrapper for the api query but is called from multiple places
4540    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4541    * but it is unclear as to whehter that is historical or there is a reason
4542    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4543    *
4544    * @param array $params
4545    * @param array $returnProperties
4546    * @param null $fields
4547    * @param string $sort
4548    * @param int $offset
4549    * @param int $row_count
4550    * @param bool $smartGroupCache
4551    *   ?? update smart group cache?.
4552    * @param bool $count
4553    *   Return count obnly.
4554    * @param bool $skipPermissions
4555    *   Should permissions be ignored or should the logged in user's permissions be applied.
4556    * @param int $mode
4557    *   This basically correlates to the component.
4558    * @param string $apiEntity
4559    *   The api entity being called.
4560    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4561    *
4562    * @param bool|null $primaryLocationOnly
4563    *
4564    * @return array
4565    * @throws \CRM_Core_Exception
4566    */
4567   public static function apiQuery(
4568     $params = NULL,
4569     $returnProperties = NULL,
4570     $fields = NULL,
4571     $sort = NULL,
4572     $offset = 0,
4573     $row_count = 25,
4574     $smartGroupCache = TRUE,
4575     $count = FALSE,
4576     $skipPermissions = TRUE,
4577     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4578     $apiEntity = NULL,
4579     $primaryLocationOnly = NULL
4580   ) {
4581
4582     $query = new CRM_Contact_BAO_Query(
4583       $params, $returnProperties,
4584       NULL, TRUE, FALSE, $mode,
4585       $skipPermissions,
4586       TRUE, $smartGroupCache,
4587       NULL, 'AND',
4588       $apiEntity, $primaryLocationOnly
4589     );
4590
4591     //this should add a check for view deleted if permissions are enabled
4592     if ($skipPermissions) {
4593       $query->_skipDeleteClause = TRUE;
4594     }
4595     $query->generatePermissionClause(FALSE, $count);
4596
4597     // note : this modifies _fromClause and _simpleFromClause
4598     $query->includePseudoFieldsJoin($sort);
4599
4600     list($select, $from, $where, $having) = $query->query($count);
4601
4602     if (!empty($query->_permissionWhereClause)) {
4603       if (!empty($query->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
4604         $from .= " $query->_permissionFromClause";
4605       }
4606       if (empty($where)) {
4607         $where = "WHERE $query->_permissionWhereClause";
4608       }
4609       else {
4610         $where = "$where AND $query->_permissionWhereClause";
4611       }
4612     }
4613
4614     $sql = "$select $from $where $having";
4615
4616     // add group by only when API action is not getcount
4617     //  otherwise query fetches incorrect count
4618     if ($query->_useGroupBy && !$count) {
4619       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4620     }
4621     if (!empty($sort)) {
4622       $sort = CRM_Utils_Type::escape($sort, 'String');
4623       $sql .= " ORDER BY $sort ";
4624     }
4625     if ($row_count > 0 && $offset >= 0) {
4626       $offset = CRM_Utils_Type::escape($offset, 'Int');
4627       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4628       $sql .= " LIMIT $offset, $row_count ";
4629     }
4630
4631     $dao = CRM_Core_DAO::executeQuery($sql);
4632
4633     // @todo derive this from the component class rather than hard-code two options.
4634     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4635
4636     $values = [];
4637     while ($dao->fetch()) {
4638       if ($count) {
4639         $noRows = $dao->rowCount;
4640         return [$noRows, NULL];
4641       }
4642       $val = $query->store($dao);
4643       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4644
4645       if (!empty($convertedVals)) {
4646         $val = array_replace_recursive($val, $convertedVals);
4647       }
4648       $values[$dao->$entityIDField] = $val;
4649     }
4650     return [$values];
4651   }
4652
4653   /**
4654    * Get the actual custom field name by stripping off the appended string.
4655    *
4656    * The string could be _relative, _from, or _to
4657    *
4658    * @todo use metadata rather than convention to do this.
4659    *
4660    * @param string $parameterName
4661    *   The name of the parameter submitted to the form.
4662    *   e.g
4663    *   custom_3_relative
4664    *   custom_3_from
4665    *
4666    * @return string
4667    */
4668   public static function getCustomFieldName($parameterName) {
4669     if (substr($parameterName, -5, 5) == '_from') {
4670       return substr($parameterName, 0, strpos($parameterName, '_from'));
4671     }
4672     if (substr($parameterName, -9, 9) == '_relative') {
4673       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4674     }
4675     if (substr($parameterName, -3, 3) == '_to') {
4676       return substr($parameterName, 0, strpos($parameterName, '_to'));
4677     }
4678   }
4679
4680   /**
4681    * Convert submitted values for relative custom fields to query object format.
4682    *
4683    * The query will support the sqlOperator format so convert to that format.
4684    *
4685    * @param array $formValues
4686    *   Submitted values.
4687    * @param array $params
4688    *   Converted parameters for the query object.
4689    * @param string $values
4690    *   Submitted value.
4691    * @param string $fieldName
4692    *   Submitted field name. (Matches form field not DB field.)
4693    */
4694   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4695     if (empty($values)) {
4696       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4697       // this function gets called again after they fields have been converted which can get ugly.
4698       return;
4699     }
4700     $customFieldName = self::getCustomFieldName($fieldName);
4701
4702     if (substr($fieldName, -9, 9) == '_relative') {
4703       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4704     }
4705     else {
4706       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4707         // Both to & from are set. We only need to acton one, choosing from.
4708         return;
4709       }
4710
4711       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4712       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4713
4714       if (self::isCustomDateField($customFieldName)) {
4715         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4716       }
4717     }
4718
4719     if ($from) {
4720       if ($to) {
4721         $relativeFunction = ['BETWEEN' => [$from, $to]];
4722       }
4723       else {
4724         $relativeFunction = ['>=' => $from];
4725       }
4726     }
4727     else {
4728       $relativeFunction = ['<=' => $to];
4729     }
4730     $params[] = [
4731       $customFieldName,
4732       '=',
4733       $relativeFunction,
4734       0,
4735       0,
4736     ];
4737   }
4738
4739   /**
4740    * Are we dealing with custom field of type date.
4741    *
4742    * @param $fieldName
4743    *
4744    * @return bool
4745    * @throws \CiviCRM_API3_Exception
4746    */
4747   public static function isCustomDateField($fieldName) {
4748     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4749       return FALSE;
4750     }
4751     try {
4752       $customFieldDataType = civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type']);
4753       if ('Date' == $customFieldDataType) {
4754         return TRUE;
4755       }
4756     }
4757     catch (CiviCRM_API3_Exception $e) {
4758     }
4759     return FALSE;
4760   }
4761
4762   /**
4763    * Has this field already been reformatting to Query object syntax.
4764    *
4765    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4766    * to come with associated double queries & is possibly damaging performance.
4767    *
4768    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4769    * them as they are.
4770    *
4771    * @param mixed $values
4772    *   Value in formValues for the field.
4773    *
4774    * @return bool;
4775    */
4776   public static function isAlreadyProcessedForQueryFormat($values) {
4777     if (!is_array($values)) {
4778       return FALSE;
4779     }
4780     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4781       return FALSE;
4782     }
4783     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4784   }
4785
4786   /**
4787    * If the state and country are passed remove state.
4788    *
4789    * Country is implicit from the state, but including both results in
4790    * a poor query as there is no combined index on state AND country.
4791    *
4792    * @see https://issues.civicrm.org/jira/browse/CRM-18125
4793    *
4794    * @param array $formValues
4795    */
4796   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4797     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4798       // The use of array map sanitises the data by ensuring we are dealing with integers.
4799       $states = implode(', ', array_map('intval', $formValues['state_province']));
4800       $countryList = CRM_Core_DAO::singleValueQuery(
4801         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4802       );
4803       if ($countryList == $formValues['country']) {
4804         unset($formValues['country']);
4805       }
4806     }
4807   }
4808
4809   /**
4810    * For some special cases, grouping by subset of select fields becomes mandatory.
4811    * Hence, full_group_by mode is handled by appending any_value
4812    * keyword to select fields not present in groupBy
4813    *
4814    * @param array $selectClauses
4815    * @param array $groupBy - Columns already included in GROUP By clause.
4816    * @param string $aggregateFunction
4817    *
4818    * @return string
4819    */
4820   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4821     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4822       $groupBy = array_map('trim', (array) $groupBy);
4823       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4824       foreach ($selectClauses as $key => &$val) {
4825         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4826         // append ANY_VALUE() keyword
4827         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4828           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4829             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4830             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4831         }
4832       }
4833     }
4834
4835     return "SELECT " . implode(', ', $selectClauses) . " ";
4836   }
4837
4838   /**
4839    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4840    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4841    * keyword to select fields not present in groupBy
4842    *
4843    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4844    * @param array $orderBys - ORDER BY sub-clauses
4845    *
4846    */
4847   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4848     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4849       foreach ($orderBys as $orderBy) {
4850         // remove sort syntax from ORDER BY clauses if present
4851         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4852         // if ORDER BY column is not present in GROUP BY then append it to end
4853         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4854           $groupBy .= ", {$orderBy}";
4855         }
4856       }
4857     }
4858   }
4859
4860   /**
4861    * Include Select columns in groupBy clause.
4862    *
4863    * @param array $selectClauses
4864    * @param array $groupBy - Columns already included in GROUP By clause.
4865    *
4866    * @return string
4867    */
4868   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4869     $groupBy = (array) $groupBy;
4870     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4871     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4872
4873     //return if ONLY_FULL_GROUP_BY is not enabled.
4874     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4875       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4876       foreach ($selectClauses as $key => $val) {
4877         $aliasArray = preg_split('/ as /i', $val);
4878         // if more than 1 alias we need to split by ','.
4879         if (count($aliasArray) > 2) {
4880           $aliasArray = preg_split('/,/', $val);
4881           foreach ($aliasArray as $key => $value) {
4882             $alias = current(preg_split('/ as /i', $value));
4883             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4884               $groupBy[] = $alias;
4885             }
4886           }
4887         }
4888         else {
4889           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4890           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4891           $tableName = current(explode('.', $selectColumn));
4892           $primaryKey = "{$tableName}.id";
4893           // exclude columns which are already included in groupBy and aggregate functions from select
4894           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4895           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4896             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4897               $groupBy[] = $alias;
4898             }
4899             else {
4900               $groupBy[] = $selectColumn;
4901             }
4902           }
4903         }
4904       }
4905     }
4906
4907     if (!empty($groupBy)) {
4908       return " GROUP BY " . implode(', ', $groupBy);
4909     }
4910     return '';
4911   }
4912
4913   /**
4914    * Create and query the db for an contact search.
4915    *
4916    * @param int $offset
4917    *   The offset for the query.
4918    * @param int $rowCount
4919    *   The number of rows to return.
4920    * @param string|CRM_Utils_Sort $sort
4921    *   The order by string.
4922    * @param bool $count
4923    *   Is this a count only query ?.
4924    * @param bool $includeContactIds
4925    *   Should we include contact ids?.
4926    * @param bool $sortByChar
4927    *   If true returns the distinct array of first characters for search results.
4928    * @param bool $groupContacts
4929    *   If true, return only the contact ids.
4930    * @param bool $returnQuery
4931    *   Should we return the query as a string.
4932    * @param string $additionalWhereClause
4933    *   If the caller wants to further restrict the search (used for components).
4934    * @param null $sortOrder
4935    * @param string $additionalFromClause
4936    *   Should be clause with proper joins, effective to reduce where clause load.
4937    *
4938    * @param bool $skipOrderAndLimit
4939    *
4940    * @return CRM_Core_DAO
4941    */
4942   public function searchQuery(
4943     $offset = 0, $rowCount = 0, $sort = NULL,
4944     $count = FALSE, $includeContactIds = FALSE,
4945     $sortByChar = FALSE, $groupContacts = FALSE,
4946     $returnQuery = FALSE,
4947     $additionalWhereClause = NULL, $sortOrder = NULL,
4948     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4949   ) {
4950
4951     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4952
4953     if ($returnQuery) {
4954       return $query;
4955     }
4956     if ($count) {
4957       return CRM_Core_DAO::singleValueQuery($query);
4958     }
4959
4960     $dao = CRM_Core_DAO::executeQuery($query);
4961
4962     // We can always call this - it will only re-enable if it was originally enabled.
4963     CRM_Core_DAO::reenableFullGroupByMode();
4964
4965     if ($groupContacts) {
4966       $ids = [];
4967       while ($dao->fetch()) {
4968         $ids[] = $dao->id;
4969       }
4970       return implode(',', $ids);
4971     }
4972
4973     return $dao;
4974   }
4975
4976   /**
4977    * Create and query the db for the list of all first letters used by contacts
4978    *
4979    * @return CRM_Core_DAO
4980    */
4981   public function alphabetQuery() {
4982     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
4983     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
4984       {$sqlParts['from']}
4985       {$sqlParts['where']}";
4986     $dao = CRM_Core_DAO::executeQuery($query);
4987     return $dao;
4988   }
4989
4990   /**
4991    * Fetch a list of contacts for displaying a search results page
4992    *
4993    * @param array $cids
4994    *   List of contact IDs
4995    * @param bool $includeContactIds
4996    * @return CRM_Core_DAO
4997    */
4998   public function getCachedContacts($cids, $includeContactIds) {
4999     CRM_Core_DAO::disableFullGroupByMode();
5000     CRM_Utils_Type::validateAll($cids, 'Positive');
5001     $this->_includeContactIds = $includeContactIds;
5002     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5003     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5004     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5005     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5006     $order = 'ORDER BY _wgt';
5007     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5008     $limit = '';
5009     $query = "$select $from $where $groupBy $order $limit";
5010
5011     $result = CRM_Core_DAO::executeQuery($query);
5012     CRM_Core_DAO::reenableFullGroupByMode();
5013     return $result;
5014   }
5015
5016   /**
5017    * Construct a SQL CASE expression.
5018    *
5019    * @param string $idCol
5020    *   The name of a column with ID's (eg 'contact_a.id').
5021    * @param array $cids
5022    *   Array(int $weight => int $id).
5023    * @return string
5024    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5025    */
5026   private function createSqlCase($idCol, $cids) {
5027     $buf = "CASE\n";
5028     foreach ($cids as $weight => $cid) {
5029       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5030     }
5031     $buf .= "END\n";
5032     return $buf;
5033   }
5034
5035   /**
5036    * Populate $this->_permissionWhereClause with permission related clause and update other
5037    * query related properties.
5038    *
5039    * Function calls ACL permission class and hooks to filter the query appropriately
5040    *
5041    * Note that these 2 params were in the code when extracted from another function
5042    * and a second round extraction would be to make them properties of the class
5043    *
5044    * @param bool $onlyDeleted
5045    *   Only get deleted contacts.
5046    * @param bool $count
5047    *   Return Count only.
5048    */
5049   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5050     if (!$this->_skipPermission) {
5051       $permissionClauses = CRM_Contact_BAO_Contact_Permission::cacheClause();
5052       $this->_permissionWhereClause = $permissionClauses[1];
5053       $this->_permissionFromClause = $permissionClauses[0];
5054
5055       if (CRM_Core_Permission::check('access deleted contacts')) {
5056         if (!$onlyDeleted) {
5057           $this->_permissionWhereClause .= ' AND (contact_a.is_deleted = 0)';
5058         }
5059         else {
5060           $this->_permissionWhereClause .= " AND (contact_a.is_deleted) ";
5061         }
5062       }
5063
5064       if (isset($this->_tables['civicrm_activity'])) {
5065         $bao = new CRM_Activity_BAO_Activity();
5066         $clauses = $subclauses = [];
5067         foreach ((array) $bao->addSelectWhereClause() as $field => $vals) {
5068           if ($vals && $field !== 'id') {
5069             $clauses[] = $bao->tableName() . ".$field " . $vals;
5070           }
5071           elseif ($vals) {
5072             $subclauses[] = "$field " . implode(" AND $field ", (array) $vals);
5073           }
5074         }
5075         if ($subclauses) {
5076           $clauses[] = $bao->tableName() . '.`id` IN (SELECT `id` FROM `' . $bao->tableName() . '` WHERE ' . implode(' AND ', $subclauses) . ')';
5077         }
5078         if (!empty($clauses) && $this->_permissionWhereClause) {
5079           $this->_permissionWhereClause .= ' AND (' . implode(' AND ', $clauses) . ')';
5080         }
5081         elseif (!empty($clauses)) {
5082           $this->_permissionWhereClause .= '(' . implode(' AND ', $clauses) . ')';
5083         }
5084       }
5085     }
5086     else {
5087       // add delete clause if needed even if we are skipping permission
5088       // CRM-7639
5089       if (!$this->_skipDeleteClause) {
5090         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5091           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5092         }
5093         else {
5094           // CRM-6181
5095           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5096         }
5097       }
5098     }
5099   }
5100
5101   /**
5102    * @param $val
5103    */
5104   public function setSkipPermission($val) {
5105     $this->_skipPermission = $val;
5106   }
5107
5108   /**
5109    * @param null $context
5110    *
5111    * @return array
5112    * @throws \CRM_Core_Exception
5113    */
5114   public function summaryContribution($context = NULL) {
5115     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5116     if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5117       $from .= " $this->_permissionFromClause";
5118     }
5119     if ($this->_permissionWhereClause) {
5120       $where .= " AND " . $this->_permissionWhereClause;
5121     }
5122     if ($context == 'search') {
5123       $where .= " AND contact_a.is_deleted = 0 ";
5124     }
5125
5126     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5127
5128     $summary = ['total' => []];
5129     $this->addBasicStatsToSummary($summary, $where, $from);
5130
5131     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5132       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5133     }
5134
5135     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5136
5137     return $summary;
5138   }
5139
5140   /**
5141    * Append financial ACL limits to the query from & where clauses, if applicable.
5142    *
5143    * @param string $where
5144    * @param string $from
5145    */
5146   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5147     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5148       return;
5149     }
5150     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5151     if (!empty($financialTypes)) {
5152       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5153       $from .= " LEFT JOIN civicrm_line_item li
5154                   ON civicrm_contribution.id = li.contribution_id AND
5155                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5156     }
5157     else {
5158       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5159     }
5160   }
5161
5162   /**
5163    * Getter for the qill object.
5164    *
5165    * @return array
5166    */
5167   public function qill() {
5168     return $this->_qill;
5169   }
5170
5171   /**
5172    * Default set of return default hier return properties.
5173    *
5174    * @return array
5175    */
5176   public static function &defaultHierReturnProperties() {
5177     if (!isset(self::$_defaultHierReturnProperties)) {
5178       self::$_defaultHierReturnProperties = [
5179         'home_URL' => 1,
5180         'image_URL' => 1,
5181         'legal_identifier' => 1,
5182         'external_identifier' => 1,
5183         'contact_type' => 1,
5184         'contact_sub_type' => 1,
5185         'sort_name' => 1,
5186         'display_name' => 1,
5187         'nick_name' => 1,
5188         'first_name' => 1,
5189         'middle_name' => 1,
5190         'last_name' => 1,
5191         'prefix_id' => 1,
5192         'suffix_id' => 1,
5193         'formal_title' => 1,
5194         'communication_style_id' => 1,
5195         'email_greeting' => 1,
5196         'postal_greeting' => 1,
5197         'addressee' => 1,
5198         'birth_date' => 1,
5199         'gender_id' => 1,
5200         'preferred_communication_method' => 1,
5201         'do_not_phone' => 1,
5202         'do_not_email' => 1,
5203         'do_not_mail' => 1,
5204         'do_not_sms' => 1,
5205         'do_not_trade' => 1,
5206         'location' => [
5207           '1' => [
5208             'location_type' => 1,
5209             'street_address' => 1,
5210             'city' => 1,
5211             'state_province' => 1,
5212             'postal_code' => 1,
5213             'postal_code_suffix' => 1,
5214             'country' => 1,
5215             'phone-Phone' => 1,
5216             'phone-Mobile' => 1,
5217             'phone-Fax' => 1,
5218             'phone-1' => 1,
5219             'phone-2' => 1,
5220             'phone-3' => 1,
5221             'im-1' => 1,
5222             'im-2' => 1,
5223             'im-3' => 1,
5224             'email-1' => 1,
5225             'email-2' => 1,
5226             'email-3' => 1,
5227           ],
5228           '2' => [
5229             'location_type' => 1,
5230             'street_address' => 1,
5231             'city' => 1,
5232             'state_province' => 1,
5233             'postal_code' => 1,
5234             'postal_code_suffix' => 1,
5235             'country' => 1,
5236             'phone-Phone' => 1,
5237             'phone-Mobile' => 1,
5238             'phone-1' => 1,
5239             'phone-2' => 1,
5240             'phone-3' => 1,
5241             'im-1' => 1,
5242             'im-2' => 1,
5243             'im-3' => 1,
5244             'email-1' => 1,
5245             'email-2' => 1,
5246             'email-3' => 1,
5247           ],
5248         ],
5249       ];
5250     }
5251     return self::$_defaultHierReturnProperties;
5252   }
5253
5254   /**
5255    * Build query for a date field.
5256    *
5257    * @param array $values
5258    * @param string $tableName
5259    * @param string $fieldName
5260    * @param string $dbFieldName
5261    * @param string $fieldTitle
5262    * @param bool $appendTimeStamp
5263    * @param string $dateFormat
5264    * @param string|null $highDBFieldName
5265    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5266    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5267    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5268    *   date & less than the end date.
5269    *
5270    * @throws \CRM_Core_Exception
5271    */
5272   public function dateQueryBuilder(
5273     $values, $tableName, $fieldName,
5274     $dbFieldName, $fieldTitle,
5275     $appendTimeStamp = TRUE,
5276     $dateFormat = 'YmdHis',
5277     $highDBFieldName = NULL
5278   ) {
5279     // @todo - remove dateFormat - pretty sure it's never passed in...
5280     list($name, $op, $value, $grouping, $wildcard) = $values;
5281     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5282       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5283       return;
5284     }
5285     if ($tableName === 'civicrm_contact') {
5286       // Special handling for contact table as it has a known alias in advanced search.
5287       $tableName = 'contact_a';
5288     }
5289     if ($name === "{$fieldName}_low" ||
5290       $name === "{$fieldName}_high"
5291     ) {
5292       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5293         return;
5294       }
5295       $this->_rangeCache[$fieldName] = 1;
5296
5297       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5298
5299       if ($name == $fieldName . '_low') {
5300         $firstOP = '>=';
5301         $firstPhrase = ts('greater than or equal to');
5302         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5303
5304         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5305         if (!empty($secondValues) && $secondValues[2]) {
5306           $secondOP = '<=';
5307           $secondPhrase = ts('less than or equal to');
5308           $secondValue = $secondValues[2];
5309
5310           if ($appendTimeStamp && strlen($secondValue) == 10) {
5311             $secondValue .= ' 23:59:59';
5312           }
5313           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5314         }
5315       }
5316       elseif ($name == $fieldName . '_high') {
5317         $firstOP = '<=';
5318         $firstPhrase = ts('less than or equal to');
5319
5320         if ($appendTimeStamp && strlen($value) == 10) {
5321           $value .= ' 23:59:59';
5322         }
5323         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5324
5325         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5326         if (!empty($secondValues) && $secondValues[2]) {
5327           $secondOP = '>=';
5328           $secondPhrase = ts('greater than or equal to');
5329           $secondValue = $secondValues[2];
5330           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5331         }
5332       }
5333
5334       if (!$appendTimeStamp) {
5335         $firstDate = substr($firstDate, 0, 8);
5336       }
5337       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5338
5339       if ($secondDate) {
5340         if (!$appendTimeStamp) {
5341           $secondDate = substr($secondDate, 0, 8);
5342         }
5343         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5344       }
5345
5346       if ($secondDate) {
5347         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5348         $this->_where[$grouping][] = "
5349 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5350 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5351 ";
5352         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5353       }
5354       else {
5355         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5356         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5357       }
5358     }
5359
5360     if ($name == $fieldName) {
5361       //In Get API, for operators other then '=' the $value is in array(op => value) format
5362       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5363         $op = key($value);
5364         $value = $value[$op];
5365       }
5366
5367       $date = $format = NULL;
5368       if (strstr($op, 'IN')) {
5369         $format = [];
5370         foreach ($value as &$date) {
5371           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5372           if (!$appendTimeStamp) {
5373             $date = substr($date, 0, 8);
5374           }
5375           $format[] = CRM_Utils_Date::customFormat($date);
5376         }
5377         $date = "('" . implode("','", $value) . "')";
5378         $format = implode(', ', $format);
5379       }
5380       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5381         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5382         if (!$appendTimeStamp) {
5383           $date = substr($date, 0, 8);
5384         }
5385         $format = CRM_Utils_Date::customFormat($date);
5386         $date = "'$date'";
5387       }
5388
5389       if ($date) {
5390         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5391       }
5392       else {
5393         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5394       }
5395
5396       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5397       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5398     }
5399
5400     // Ensure the tables are set, but don't whomp anything.
5401     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5402     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5403   }
5404
5405   /**
5406    * @param $values
5407    * @param string $tableName
5408    * @param string $fieldName
5409    * @param string $dbFieldName
5410    * @param $fieldTitle
5411    * @param null $options
5412    */
5413   public function numberRangeBuilder(
5414     &$values,
5415     $tableName, $fieldName,
5416     $dbFieldName, $fieldTitle,
5417     $options = NULL
5418   ) {
5419     list($name, $op, $value, $grouping, $wildcard) = $values;
5420
5421     if ($name == "{$fieldName}_low" ||
5422       $name == "{$fieldName}_high"
5423     ) {
5424       if (isset($this->_rangeCache[$fieldName])) {
5425         return;
5426       }
5427       $this->_rangeCache[$fieldName] = 1;
5428
5429       $secondOP = $secondPhrase = $secondValue = NULL;
5430
5431       if ($name == "{$fieldName}_low") {
5432         $firstOP = '>=';
5433         $firstPhrase = ts('greater than');
5434
5435         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5436         if (!empty($secondValues)) {
5437           $secondOP = '<=';
5438           $secondPhrase = ts('less than');
5439           $secondValue = $secondValues[2];
5440         }
5441       }
5442       else {
5443         $firstOP = '<=';
5444         $firstPhrase = ts('less than');
5445
5446         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5447         if (!empty($secondValues)) {
5448           $secondOP = '>=';
5449           $secondPhrase = ts('greater than');
5450           $secondValue = $secondValues[2];
5451         }
5452       }
5453
5454       if ($secondOP) {
5455         $this->_where[$grouping][] = "
5456 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5457 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5458 ";
5459         $displayValue = $options ? $options[$value] : $value;
5460         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5461
5462         $this->_qill[$grouping][]
5463           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5464       }
5465       else {
5466         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5467         $displayValue = $options ? $options[$value] : $value;
5468         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5469       }
5470       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5471
5472       return;
5473     }
5474
5475     if ($name == $fieldName) {
5476       $op = '=';
5477       $phrase = '=';
5478
5479       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5480
5481       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5482       $displayValue = $options ? $options[$value] : $value;
5483       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5484     }
5485   }
5486
5487   /**
5488    * @param $values
5489    * @param string $tableName
5490    * @param string $fieldName
5491    * @param string $dbFieldName
5492    * @param $fieldTitle
5493    * @param null $options
5494    */
5495   public function ageRangeQueryBuilder(
5496     &$values,
5497     $tableName, $fieldName,
5498     $dbFieldName, $fieldTitle,
5499     $options = NULL
5500   ) {
5501     list($name, $op, $value, $grouping, $wildcard) = $values;
5502
5503     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5504     // will be treated as current day
5505     $asofDate = NULL;
5506     if ($asofDateValues) {
5507       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5508       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5509       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5510     }
5511
5512     if ($name == "{$fieldName}_low" ||
5513       $name == "{$fieldName}_high"
5514     ) {
5515       if (isset($this->_rangeCache[$fieldName])) {
5516         return;
5517       }
5518       $this->_rangeCache[$fieldName] = 1;
5519
5520       $secondOP = $secondPhrase = $secondValue = NULL;
5521
5522       if ($name == "{$fieldName}_low") {
5523         $firstPhrase = ts('greater than or equal to');
5524         // NB: age > X means date of birth < Y
5525         $firstOP = '<=';
5526         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5527
5528         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5529         if (!empty($secondValues)) {
5530           $secondOP = '>=';
5531           $secondPhrase = ts('less than or equal to');
5532           $secondValue = $secondValues[2];
5533           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5534         }
5535       }
5536       else {
5537         $firstOP = '>=';
5538         $firstPhrase = ts('less than or equal to');
5539         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5540
5541         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5542         if (!empty($secondValues)) {
5543           $secondOP = '<=';
5544           $secondPhrase = ts('greater than or equal to');
5545           $secondValue = $secondValues[2];
5546           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5547         }
5548       }
5549
5550       if ($secondOP) {
5551         $this->_where[$grouping][] = "
5552 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5553 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5554 ";
5555         $displayValue = $options ? $options[$value] : $value;
5556         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5557
5558         $this->_qill[$grouping][]
5559           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5560       }
5561       else {
5562         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5563         $displayValue = $options ? $options[$value] : $value;
5564         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5565       }
5566       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5567       return;
5568     }
5569   }
5570
5571   /**
5572    * Calculate date from age.
5573    *
5574    * @param string $asofDate
5575    * @param int $age
5576    * @param string $type
5577    *
5578    * @return string
5579    * @throws \Exception
5580    */
5581   public static function calcDateFromAge($asofDate, $age, $type) {
5582     $date = new DateTime($asofDate);
5583     if ($type == "min") {
5584       // minimum age is $age: dob <= date - age "235959"
5585       $date->sub(new DateInterval("P" . $age . "Y"));
5586       return $date->format('Ymd') . "235959";
5587     }
5588     else {
5589       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5590       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5591       return $date->format('Ymd') . "000000";
5592     }
5593   }
5594
5595   /**
5596    * Given the field name, operator, value & its data type
5597    * builds the where Clause for the query
5598    * used for handling 'IS NULL'/'IS NOT NULL' operators
5599    *
5600    * @param string $field
5601    *   Fieldname.
5602    * @param string $op
5603    *   Operator.
5604    * @param string $value
5605    *   Value.
5606    * @param string $dataType
5607    *   Data type of the field.
5608    *
5609    * @return string
5610    *   Where clause for the query.
5611    * @throws \CRM_Core_Exception
5612    */
5613   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5614     $op = trim($op);
5615     $clause = "$field $op";
5616
5617     switch ($op) {
5618       case 'IS NULL':
5619       case 'IS NOT NULL':
5620         return $clause;
5621
5622       case 'IS EMPTY':
5623         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5624         return $clause;
5625
5626       case 'IS NOT EMPTY':
5627         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5628         return $clause;
5629
5630       case 'RLIKE':
5631         return " CAST({$field} AS BINARY) RLIKE BINARY '{$value}' ";
5632
5633       case 'IN':
5634       case 'NOT IN':
5635         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5636         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5637           $value = [$op => (array) $value];
5638         }
5639
5640       default:
5641         if (empty($dataType) || $dataType == 'Date') {
5642           $dataType = 'String';
5643         }
5644         if (is_array($value)) {
5645           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5646           // widely used and consistent across the codebase
5647           // adding this here won't accept the search functions which don't submit an array
5648           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5649
5650             return $queryString;
5651           }
5652           if (!empty($value[0]) && $op === 'BETWEEN') {
5653             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5654             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5655               return $queryString;
5656             }
5657           }
5658           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5659         }
5660
5661         $value = CRM_Utils_Type::escape($value, $dataType);
5662         // if we don't have a dataType we should assume
5663         if ($dataType == 'String' || $dataType == 'Text') {
5664           $value = "'" . $value . "'";
5665         }
5666         return "$clause $value";
5667     }
5668   }
5669
5670   /**
5671    * @param bool $reset
5672    *
5673    * @return array
5674    */
5675   public function openedSearchPanes($reset = FALSE) {
5676     if (!$reset || empty($this->_whereTables)) {
5677       return self::$_openedPanes;
5678     }
5679
5680     // pane name to table mapper
5681     $panesMapper = [
5682       ts('Contributions') => 'civicrm_contribution',
5683       ts('Memberships') => 'civicrm_membership',
5684       ts('Events') => 'civicrm_participant',
5685       ts('Relationships') => 'civicrm_relationship',
5686       ts('Activities') => 'civicrm_activity',
5687       ts('Pledges') => 'civicrm_pledge',
5688       ts('Cases') => 'civicrm_case',
5689       ts('Grants') => 'civicrm_grant',
5690       ts('Address Fields') => 'civicrm_address',
5691       ts('Notes') => 'civicrm_note',
5692       ts('Change Log') => 'civicrm_log',
5693       ts('Mailings') => 'civicrm_mailing',
5694     ];
5695     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5696
5697     foreach (array_keys($this->_whereTables) as $table) {
5698       if ($panName = array_search($table, $panesMapper)) {
5699         self::$_openedPanes[$panName] = TRUE;
5700       }
5701     }
5702
5703     return self::$_openedPanes;
5704   }
5705
5706   /**
5707    * @param $operator
5708    */
5709   public function setOperator($operator) {
5710     $validOperators = ['AND', 'OR'];
5711     if (!in_array($operator, $validOperators)) {
5712       $operator = 'AND';
5713     }
5714     $this->_operator = $operator;
5715   }
5716
5717   /**
5718    * @return string
5719    */
5720   public function getOperator() {
5721     return $this->_operator;
5722   }
5723
5724   /**
5725    * @param $from
5726    * @param $where
5727    * @param $having
5728    */
5729   public function filterRelatedContacts(&$from, &$where, &$having) {
5730     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5731       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5732     }
5733     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5734     // since there only can be one instance of this filter in every query
5735     // skip if filter has already applied
5736     foreach ($_rTempCache as $acache) {
5737       foreach ($acache['queries'] as $aqcache) {
5738         if (strpos($from, $aqcache['from']) !== FALSE) {
5739           $having = NULL;
5740           return;
5741         }
5742       }
5743     }
5744     $arg_sig = sha1("$from $where $having");
5745     if (isset($_rTempCache[$arg_sig])) {
5746       $cache = $_rTempCache[$arg_sig];
5747     }
5748     else {
5749       // create temp table with contact ids
5750
5751       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5752
5753       $sql = "
5754 REPLACE INTO $tableName ( contact_id )
5755 SELECT contact_a.id
5756        $from
5757        $where
5758        $having
5759 ";
5760       CRM_Core_DAO::executeQuery($sql);
5761
5762       $cache = ['tableName' => $tableName, 'queries' => []];
5763       $_rTempCache[$arg_sig] = $cache;
5764     }
5765     // upsert the query depending on relationship type
5766     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5767       $qcache = $cache['queries'][$this->_displayRelationshipType];
5768     }
5769     else {
5770       $tableName = $cache['tableName'];
5771       $qcache = [
5772         "from" => "",
5773         "where" => "",
5774       ];
5775       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5776       if (is_numeric($this->_displayRelationshipType)) {
5777         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5778         $qcache['from'] = "
5779 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5780 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5781 ";
5782         $qcache['where'] = "
5783 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5784 AND   displayRelType.is_active = 1
5785 ";
5786       }
5787       else {
5788         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5789         if ($dirOne == 'a') {
5790           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5791           $qcache['from'] .= "
5792 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5793 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5794 ";
5795         }
5796         else {
5797           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5798           $qcache['from'] .= "
5799 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5800 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5801 ";
5802         }
5803         $qcache['where'] = "
5804 WHERE displayRelType.relationship_type_id = $relType
5805 AND   displayRelType.is_active = 1
5806 ";
5807       }
5808       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5809       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5810     }
5811     $qillMessage = ts('Contacts with a Relationship Type of: ');
5812     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5813     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5814       $this->_qill[0][] = $iqill;
5815     }
5816     if (strpos($from, $qcache['from']) === FALSE) {
5817       if (strpos($from, "INNER JOIN") !== FALSE) {
5818         // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5819         // this happens when we have an event_type in the quert (CRM-7969)
5820         $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5821         // Make sure the relationship join right after the FROM and other joins afterwards.
5822         // This gives us the possibility to change the join on civicrm case.
5823         $from = preg_replace("/LEFT JOIN/", $qcache['from'] . " LEFT JOIN", $from, 1);
5824       }
5825       else {
5826         $from .= $qcache['from'];
5827       }
5828       $where = $qcache['where'];
5829       if (!empty($this->_tables['civicrm_case'])) {
5830         // Change the join on CiviCRM case so that it joins on the right contac from the relationship.
5831         $from = str_replace("ON civicrm_case_contact.contact_id = contact_a.id", "ON civicrm_case_contact.contact_id = transform_temp.contact_id", $from);
5832         $where .= " AND displayRelType.case_id = civicrm_case_contact.case_id ";
5833       }
5834       if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5835         $from .= " $this->_permissionFromClause";
5836       }
5837       if (!empty($this->_permissionWhereClause)) {
5838         $where .= "AND $this->_permissionWhereClause";
5839       }
5840     }
5841
5842     $having = NULL;
5843   }
5844
5845   /**
5846    * See CRM-19811 for why this is database hurty without apparent benefit.
5847    *
5848    * @param $op
5849    *
5850    * @return bool
5851    */
5852   public static function caseImportant($op) {
5853     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5854   }
5855
5856   /**
5857    * @param $returnProperties
5858    * @param $prefix
5859    *
5860    * @return bool
5861    */
5862   public static function componentPresent(&$returnProperties, $prefix) {
5863     foreach ($returnProperties as $name => $dontCare) {
5864       if (substr($name, 0, strlen($prefix)) == $prefix) {
5865         return TRUE;
5866       }
5867     }
5868     return FALSE;
5869   }
5870
5871   /**
5872    * Builds the necessary structures for all fields that are similar to option value look-ups.
5873    *
5874    * @param string $name
5875    *   the name of the field.
5876    * @param string $op
5877    *   the sql operator, this function should handle ALL SQL operators.
5878    * @param string $value
5879    *   depends on the operator and who's calling the query builder.
5880    * @param int $grouping
5881    *   the index where to place the where clause.
5882    * @param string $daoName
5883    *   DAO Name.
5884    * @param array $field
5885    *   an array that contains various properties of the field identified by $name.
5886    * @param string $label
5887    *   The label for this field element.
5888    * @param string $dataType
5889    *
5890    * @throws \CRM_Core_Exception
5891    */
5892   public function optionValueQuery(
5893     $name,
5894     $op,
5895     $value,
5896     $grouping,
5897     $daoName = NULL,
5898     $field,
5899     $label,
5900     $dataType = 'String'
5901   ) {
5902
5903     $pseudoFields = [
5904       'email_greeting',
5905       'postal_greeting',
5906       'addressee',
5907     ];
5908
5909     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5910     if ($tableName == 'civicrm_contact') {
5911       $wc = "contact_a.$fieldName";
5912     }
5913     else {
5914       // Special handling for on_hold, so that we actually use the 'where'
5915       // property in order to limit the query by the on_hold status of the email,
5916       // instead of using email.id which would be nonsensical.
5917       if ($field['name'] === 'on_hold') {
5918         $wc = $field['where'];
5919       }
5920       else {
5921         $wc = "$tableName.id";
5922       }
5923     }
5924
5925     if (in_array($name, $pseudoFields)) {
5926       $wc = "contact_a.{$name}_id";
5927       $dataType = 'Positive';
5928       $value = (!$value) ? 0 : $value;
5929     }
5930     if ($name == "world_region") {
5931       $field['name'] = $name;
5932     }
5933
5934     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5935     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5936     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5937   }
5938
5939   /**
5940    * Check and explode a user defined numeric string into an array
5941    * this was the protocol used by search builder in the old old days before we had
5942    * super nice js widgets to do the hard work
5943    *
5944    * @param string $string
5945    * @param string $dataType
5946    *   The dataType we should check for the values, default integer.
5947    *
5948    * @return bool|array
5949    *   false if string does not match the pattern
5950    *   array of numeric values if string does match the pattern
5951    */
5952   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5953     $string = trim($string);
5954     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5955       return FALSE;
5956     }
5957
5958     $string = substr($string, 1, -1);
5959     $values = explode(',', $string);
5960     if (empty($values)) {
5961       return FALSE;
5962     }
5963
5964     $returnValues = [];
5965     foreach ($values as $v) {
5966       if ($dataType == 'Integer' && !is_numeric($v)) {
5967         return FALSE;
5968       }
5969       elseif ($dataType == 'String' && !is_string($v)) {
5970         return FALSE;
5971       }
5972       $returnValues[] = trim($v);
5973     }
5974
5975     if (empty($returnValues)) {
5976       return FALSE;
5977     }
5978
5979     return $returnValues;
5980   }
5981
5982   /**
5983    * Convert the pseudo constants id's to their names
5984    *
5985    * @param CRM_Core_DAO $dao
5986    * @param bool $return
5987    * @param bool $usedForAPI
5988    *
5989    * @return array|NULL
5990    */
5991   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
5992     if (empty($this->_pseudoConstantsSelect)) {
5993       return NULL;
5994     }
5995     $values = [];
5996     foreach ($this->_pseudoConstantsSelect as $key => $value) {
5997       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
5998         continue;
5999       }
6000
6001       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6002         $val = $dao->{$value['idCol']};
6003         if ($key == 'groups') {
6004           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6005           continue;
6006         }
6007
6008         if (CRM_Utils_System::isNull($val)) {
6009           $dao->$key = NULL;
6010         }
6011         elseif (!empty($value['pseudoconstant'])) {
6012           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6013           // and we can use the metadata to figure it out.
6014           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6015           // more metadata based choices.
6016           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6017             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6018             foreach ($dbValues as $pseudoValue) {
6019               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6020             }
6021
6022             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6023             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
6024             if ($usedForAPI && $realFieldName) {
6025               // normally we would see 2 fields returned for pseudoConstants. An exception is
6026               // preferred_communication_method where there is no id-variant.
6027               // For the api we prioritise getting the real data returned.
6028               // over the resolved version
6029               $dao->$realFieldName = $dbValues;
6030             }
6031
6032           }
6033           else {
6034             // This is basically the same as the default but since we have the bao we can use
6035             // a cached function.
6036             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6037           }
6038         }
6039         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6040           //preserve id value
6041           $idColumn = "{$key}_id";
6042           $dao->$idColumn = $val;
6043
6044           if ($key == 'state_province_name') {
6045             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6046           }
6047           else {
6048             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6049           }
6050         }
6051         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6052           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6053         }
6054         // @todo handle this in the section above for pseudoconstants.
6055         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6056           // @todo define bao on this & merge into the above condition.
6057           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6058
6059           if ($value['pseudoField'] == 'participant_role') {
6060             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6061             foreach ($viewValues as $k => $v) {
6062               $viewValues[$k] = $pseudoOptions[$v];
6063             }
6064           }
6065           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6066         }
6067         else {
6068           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6069           $dao->$key = $labels[$val];
6070         }
6071
6072         // return converted values in array format
6073         if ($return) {
6074           if (strpos($key, '-') !== FALSE) {
6075             $keyVal = explode('-', $key);
6076             $current = &$values;
6077             $lastElement = array_pop($keyVal);
6078             foreach ($keyVal as $v) {
6079               if (!array_key_exists($v, $current)) {
6080                 $current[$v] = [];
6081               }
6082               $current = &$current[$v];
6083             }
6084             $current[$lastElement] = $dao->$key;
6085           }
6086           else {
6087             $values[$key] = $dao->$key;
6088           }
6089         }
6090       }
6091     }
6092     if (!$usedForAPI) {
6093       foreach ($this->legacyHackedFields as $realField => $labelField) {
6094         // This is a temporary routine for handling these fields while
6095         // we figure out how to handled them based on metadata in
6096         /// export and search builder. CRM-19815, CRM-19830.
6097         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6098           $dao->$realField = $dao->$labelField;
6099         }
6100       }
6101     }
6102     return $values;
6103   }
6104
6105   /**
6106    * Include pseudo fields LEFT JOIN.
6107    * @param string|array $sort can be a object or string
6108    *
6109    * @return array|NULL
6110    */
6111   public function includePseudoFieldsJoin($sort) {
6112     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6113       return NULL;
6114     }
6115     $sort = is_string($sort) ? $sort : $sort->orderBy();
6116     $present = [];
6117
6118     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6119       if (!empty($value['table'])) {
6120         $regex = "/({$value['table']}\.|{$name})/";
6121         if (preg_match($regex, $sort)) {
6122           $this->_elemnt[$value['element']] = 1;
6123           $this->_select[$value['element']] = $value['select'];
6124           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6125           $present[$value['table']] = $value['join'];
6126         }
6127       }
6128     }
6129     $presentSimpleFrom = $present;
6130
6131     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6132       array_key_exists('civicrm_country', $presentSimpleFrom)
6133     ) {
6134       unset($presentSimpleFrom['civicrm_country']);
6135     }
6136     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6137       array_key_exists('civicrm_country', $present)
6138     ) {
6139       unset($present['civicrm_country']);
6140     }
6141
6142     $presentClause = $presentSimpleFromClause = NULL;
6143     if (!empty($present)) {
6144       $presentClause = implode(' ', $present);
6145     }
6146     if (!empty($presentSimpleFrom)) {
6147       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6148     }
6149
6150     $this->_fromClause = $this->_fromClause . $presentClause;
6151     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6152
6153     return [$presentClause, $presentSimpleFromClause];
6154   }
6155
6156   /**
6157    * Build qill for field.
6158    *
6159    * Qill refers to the query detail visible on the UI.
6160    *
6161    * @param string $daoName
6162    * @param string $fieldName
6163    * @param mixed $fieldValue
6164    * @param string $op
6165    * @param array $pseudoExtraParam
6166    * @param int $type
6167    *   Type of the field per CRM_Utils_Type
6168    *
6169    * @return array
6170    */
6171   public static function buildQillForFieldValue(
6172     $daoName,
6173     $fieldName,
6174     $fieldValue,
6175     $op,
6176     $pseudoExtraParam = [],
6177     $type = CRM_Utils_Type::T_STRING
6178   ) {
6179     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6180
6181     //API usually have fieldValue format as array(operator => array(values)),
6182     //so we need to separate operator out of fieldValue param
6183     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6184       $op = key($fieldValue);
6185       $fieldValue = $fieldValue[$op];
6186     }
6187
6188     // if Operator chosen is NULL/EMPTY then
6189     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6190       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6191     }
6192
6193     // @todo - if the right BAO is passed in special handling for the below
6194     // fields should not be required. testQillOptions.
6195     if ($fieldName == 'country_id') {
6196       $pseudoOptions = CRM_Core_PseudoConstant::country();
6197     }
6198     elseif ($fieldName == 'county_id') {
6199       $pseudoOptions = CRM_Core_PseudoConstant::county();
6200     }
6201     elseif ($fieldName == 'world_region') {
6202       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6203     }
6204     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6205       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6206       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6207     }
6208     elseif ($fieldName == 'contribution_product_id') {
6209       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6210     }
6211     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6212       $pseudoOptions = CRM_Core_PseudoConstant::group();
6213     }
6214     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6215       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6216     }
6217     elseif ($daoName) {
6218       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6219     }
6220
6221     if (is_array($fieldValue)) {
6222       $qillString = [];
6223       if (!empty($pseudoOptions)) {
6224         foreach ((array) $fieldValue as $val) {
6225           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6226         }
6227         $fieldValue = implode(', ', $qillString);
6228       }
6229       else {
6230         if ($type == CRM_Utils_Type::T_DATE) {
6231           foreach ($fieldValue as $index => $value) {
6232             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6233           }
6234         }
6235         $separator = ', ';
6236         // @todo - this is a bit specific (one operator).
6237         // However it is covered by a unit test so can be altered later with
6238         // some confidence.
6239         if ($op === 'BETWEEN') {
6240           $separator = ' AND ';
6241         }
6242         $fieldValue = implode($separator, $fieldValue);
6243       }
6244     }
6245     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6246       $fieldValue = $pseudoOptions[$fieldValue];
6247     }
6248     elseif ($type === CRM_Utils_Type::T_DATE) {
6249       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6250     }
6251
6252     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6253   }
6254
6255   /**
6256    * Get the qill (search description for field) for the specified field.
6257    *
6258    * @param string $daoName
6259    * @param string $name
6260    * @param string $value
6261    * @param string|array $op
6262    * @param string $label
6263    *
6264    * @return string
6265    */
6266   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6267     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6268     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6269   }
6270
6271   /**
6272    * Alter value to reflect wildcard settings.
6273    *
6274    * The form will have tried to guess whether this is a good field to wildcard but there is
6275    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6276    * or only the end of the string
6277    *
6278    * @param bool $wildcard
6279    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6280    * @param string $op
6281    *   Generally '=' or 'LIKE'.
6282    * @param string $value
6283    *   The search string.
6284    *
6285    * @return string
6286    */
6287   public static function getWildCardedValue($wildcard, $op, $value) {
6288     if ($wildcard && $op === 'LIKE') {
6289       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6290         return "%$value%";
6291       }
6292       else {
6293         return "$value%";
6294       }
6295     }
6296     else {
6297       return "$value";
6298     }
6299   }
6300
6301   /**
6302    * Process special fields of Search Form in OK (Operator in Key) format
6303    *
6304    * @param array $formValues
6305    * @param array $specialFields
6306    *    Special params to be processed
6307    * @param array $changeNames
6308    *   Array of fields whose name should be changed
6309    */
6310   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6311     // Array of special fields whose value are considered only for NULL or EMPTY operators
6312     $nullableFields = ['contribution_batch_id'];
6313
6314     foreach ($specialFields as $element) {
6315       $value = $formValues[$element] ?? NULL;
6316       if ($value) {
6317         if (is_array($value)) {
6318           if (in_array($element, array_keys($changeNames))) {
6319             unset($formValues[$element]);
6320             $element = $changeNames[$element];
6321           }
6322           $formValues[$element] = ['IN' => $value];
6323         }
6324         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6325           $formValues[$element] = [$value => 1];
6326         }
6327         elseif (!in_array($element, $nullableFields)) {
6328           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6329           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6330           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6331         }
6332       }
6333     }
6334   }
6335
6336   /**
6337    * Parse and assimilate the various sort options.
6338    *
6339    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6340    * `email`), the related table may be joined automatically.
6341    *
6342    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6343    * it's been extracted to a standalone function.
6344    *
6345    * @param string|CRM_Utils_Sort $sort
6346    *   The order by string.
6347    * @param null $sortOrder
6348    *   Who knows? Hu knows. He who knows Hu knows who.
6349    *
6350    * @return string
6351    *   list(string $orderByClause, string $additionalFromClause).
6352    *
6353    * @throws \CRM_Core_Exception
6354    */
6355   protected function prepareOrderBy($sort, $sortOrder) {
6356     $orderByArray = [];
6357     $orderBy = '';
6358
6359     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6360       isset($this->_distinctComponentClause)
6361     ) {
6362       if ($sort) {
6363         if (is_string($sort)) {
6364           $orderBy = $sort;
6365         }
6366         else {
6367           $orderBy = trim($sort->orderBy());
6368         }
6369         // Deliberately remove the backticks again, as they mess up the evil
6370         // string munging below. This balanced by re-escaping before use.
6371         $orderBy = str_replace('`', '', $orderBy);
6372
6373         if (!empty($orderBy)) {
6374           // this is special case while searching for
6375           // change log CRM-1718
6376           if (preg_match('/sort_name/i', $orderBy)) {
6377             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6378           }
6379
6380           if ($sortOrder) {
6381             $orderBy .= " $sortOrder";
6382           }
6383
6384           // always add contact_a.id to the ORDER clause
6385           // so the order is deterministic
6386           if (strpos('contact_a.id', $orderBy) === FALSE) {
6387             $orderBy .= ", contact_a.id";
6388           }
6389         }
6390       }
6391       else {
6392         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6393       }
6394     }
6395     if (!$orderBy) {
6396       return NULL;
6397     }
6398     // Remove this here & add it at the end for simplicity.
6399     $order = trim($orderBy);
6400     $orderByArray = explode(',', $order);
6401
6402     foreach ($orderByArray as $orderByClause) {
6403       $orderByClauseParts = explode(' ', trim($orderByClause));
6404       $field = $orderByClauseParts[0];
6405       $direction = $orderByClauseParts[1] ?? 'asc';
6406       $fieldSpec = $this->getMetadataForRealField($field);
6407
6408       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6409       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6410       // add keys to $this->fields to represent the actual keys for locations.
6411       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6412         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6413         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6414       }
6415
6416       if ($this->_returnProperties === []) {
6417         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6418           $this->_tables[$fieldSpec['table_name']] = 1;
6419           $order = $fieldSpec['where'] . ' ' . $direction;
6420         }
6421
6422       }
6423       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6424       // add to cfIDs array if not present
6425       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6426         $this->_cfIDs[$cfID] = [];
6427         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6428         $this->_customQuery->query();
6429         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6430         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6431       }
6432
6433       // By replacing the join to the option value table with the mysql construct
6434       // ORDER BY field('contribution_status_id', 2,1,4)
6435       // we can remove a join. In the case of the option value join it is
6436       /// a join known to cause slow queries.
6437       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6438       // matching an id+name parrern in the
6439       // first instance for scope reasons. They require slightly different handling as the column (label)
6440       // is not declared for them.
6441       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6442       // escaping.
6443       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6444       if (!empty($pseudoConstantMetadata)
6445       ) {
6446         if (!empty($pseudoConstantMetadata['optionGroupName'])
6447           || $this->isPseudoFieldAnFK($fieldSpec)
6448         ) {
6449           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6450           if (empty($fieldSpec['bao'])) {
6451             continue;
6452           }
6453           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name']);
6454           natcasesort($sortedOptions);
6455           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6456           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6457           // this might give some early warning.
6458           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6459           // use where if it's set to fully qualify ambiguous column names
6460           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6461           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6462           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6463         }
6464         //CRM-12565 add "`" around $field if it is a pseudo constant
6465         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6466         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6467           $order = str_replace($field, "`{$field}`", $order);
6468         }
6469       }
6470     }
6471
6472     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6473     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6474
6475     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6476     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6477     if ($order) {
6478       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6479       return ' ORDER BY ' . $order;
6480     }
6481   }
6482
6483   /**
6484    * Convert a string of group IDs to a string of group labels.
6485    *
6486    * The original string may include duplicates and groups the user does not have
6487    * permission to see.
6488    *
6489    * @param CRM_Core_DAO $dao
6490    * @param string $val
6491    *
6492    * @return string
6493    */
6494   public function convertGroupIDStringToLabelString(&$dao, $val) {
6495     $groupIDs = explode(',', $val);
6496     // Note that groups that the user does not have permission to will be excluded (good).
6497     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6498     return implode(', ', $groups);
6499
6500   }
6501
6502   /**
6503    * Set the qill and where properties for a field.
6504    *
6505    * This function is intended as a short-term function to encourage refactoring
6506    * & re-use - but really we should just have less special-casing.
6507    *
6508    * @param string $name
6509    * @param string $op
6510    * @param string|array $value
6511    * @param string $grouping
6512    * @param array $field
6513    *
6514    * @throws \CRM_Core_Exception
6515    */
6516   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6517     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6518     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6519     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6520       1 => $field['title'],
6521       2 => $qillop,
6522       3 => $qillVal,
6523     ]);
6524   }
6525
6526   /**
6527    * Has the pseudoconstant of the field been requested.
6528    *
6529    * For example if the field is payment_instrument_id then it
6530    * has been requested if either payment_instrument_id or payment_instrument
6531    * have been requested. Payment_instrument is the option groun name field value.
6532    *
6533    * @param array $field
6534    * @param string $fieldName
6535    *   The unique name of the field - ie. the one it will be aliased to in the query.
6536    *
6537    * @return bool
6538    */
6539   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6540     $realField = $this->getMetadataForRealField($fieldName);
6541     if (!isset($realField['pseudoconstant'])) {
6542       return FALSE;
6543     }
6544     $pseudoConstant = $realField['pseudoconstant'];
6545     if (empty($pseudoConstant['optionGroupName']) &&
6546       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6547       // We are increasing our pseudoconstant handling - but still very cautiously,
6548       // hence the check for labelColumn === name
6549       return FALSE;
6550     }
6551
6552     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6553       return TRUE;
6554     }
6555     if (!empty($this->_returnProperties[$fieldName])) {
6556       return TRUE;
6557     }
6558     // Is this still required - the above goes off the unique name. Test with things like
6559     // communication_preferences & prefix_id.
6560     if (!empty($this->_returnProperties[$field['name']])) {
6561       return TRUE;
6562     }
6563     return FALSE;
6564   }
6565
6566   /**
6567    * Get Select Clause.
6568    *
6569    * @return string
6570    */
6571   public function getSelect() {
6572     $select = 'SELECT ';
6573     if (isset($this->_distinctComponentClause)) {
6574       $select .= "{$this->_distinctComponentClause}, ";
6575     }
6576     $select .= implode(', ', $this->_select);
6577     return $select;
6578   }
6579
6580   /**
6581    * Add basic statistics to the summary.
6582    *
6583    * @param array $summary
6584    * @param string $where
6585    * @param string $from
6586    *
6587    * @return array
6588    * @throws \CRM_Core_Exception
6589    */
6590   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6591     $summary['total']['count'] = 0;
6592     $summary['total']['amount'] = $summary['total']['avg'] = [];
6593
6594     $query = "
6595       SELECT COUNT( conts.total_amount ) as total_count,
6596         SUM(   conts.total_amount ) as total_amount,
6597         AVG(   conts.total_amount ) as total_avg,
6598         conts.currency              as currency
6599       FROM (
6600         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6601         civicrm_contribution.currency
6602         $from
6603         $where AND civicrm_contribution.contribution_status_id = 1
6604       GROUP BY civicrm_contribution.id
6605     ) as conts
6606     GROUP BY currency";
6607
6608     $dao = CRM_Core_DAO::executeQuery($query);
6609
6610     while ($dao->fetch()) {
6611       $summary['total']['count'] += $dao->total_count;
6612       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6613       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6614     }
6615
6616     if (!empty($summary['total']['amount'])) {
6617       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6618       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6619     }
6620     else {
6621       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6622     }
6623     return $summary;
6624   }
6625
6626   /**
6627    * Add basic soft credit statistics to summary array.
6628    *
6629    * @param array $summary
6630    * @param string $where
6631    * @param string $from
6632    *
6633    * @throws \CRM_Core_Exception
6634    */
6635   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6636     $query = "
6637       SELECT COUNT( conts.total_amount ) as total_count,
6638       SUM(   conts.total_amount ) as total_amount,
6639       AVG(   conts.total_amount ) as total_avg,
6640       conts.currency as currency
6641       FROM (
6642         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6643         $from
6644         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6645         GROUP BY civicrm_contribution_soft.id
6646       ) as conts
6647       GROUP BY currency";
6648
6649     $dao = CRM_Core_DAO::executeQuery($query);
6650     $summary['soft_credit']['count'] = 0;
6651     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6652     while ($dao->fetch()) {
6653       $summary['soft_credit']['count'] += $dao->total_count;
6654       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6655       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6656     }
6657     if (!empty($summary['soft_credit']['amount'])) {
6658       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6659       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6660     }
6661     else {
6662       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6663     }
6664   }
6665
6666   /**
6667    * Add basic stats about cancelled contributions to the summary.
6668    *
6669    * @param array $summary
6670    * @param string $where
6671    * @param string $from
6672    *
6673    * @throws \CRM_Core_Exception
6674    */
6675   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6676     $query = "
6677       SELECT COUNT( conts.total_amount ) as cancel_count,
6678        SUM(   conts.total_amount ) as cancel_amount,
6679        AVG(   conts.total_amount ) as cancel_avg,
6680        conts.currency              as currency
6681         FROM (
6682       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6683       $from
6684       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6685       GROUP BY civicrm_contribution.id
6686     ) as conts
6687     GROUP BY currency";
6688
6689     $dao = CRM_Core_DAO::executeQuery($query);
6690
6691     if ($dao->N <= 1) {
6692       if ($dao->fetch()) {
6693         $summary['cancel']['count'] = $dao->cancel_count;
6694         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6695         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6696       }
6697     }
6698     else {
6699       $summary['cancel']['count'] = 0;
6700       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6701       while ($dao->fetch()) {
6702         $summary['cancel']['count'] += $dao->cancel_count;
6703         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6704         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6705       }
6706       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6707       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6708     }
6709   }
6710
6711   /**
6712    * Create the sql query for an contact search.
6713    *
6714    * @param int $offset
6715    *   The offset for the query.
6716    * @param int $rowCount
6717    *   The number of rows to return.
6718    * @param string|CRM_Utils_Sort $sort
6719    *   The order by string.
6720    * @param bool $count
6721    *   Is this a count only query ?.
6722    * @param bool $includeContactIds
6723    *   Should we include contact ids?.
6724    * @param bool $sortByChar
6725    *   If true returns the distinct array of first characters for search results.
6726    * @param bool $groupContacts
6727    *   If true, return only the contact ids.
6728    * @param string $additionalWhereClause
6729    *   If the caller wants to further restrict the search (used for components).
6730    * @param null $sortOrder
6731    * @param string $additionalFromClause
6732    *   Should be clause with proper joins, effective to reduce where clause load.
6733    *
6734    * @param bool $skipOrderAndLimit
6735    *
6736    * @return string
6737    *
6738    * @throws \CRM_Core_Exception
6739    */
6740   public function getSearchSQL(
6741     $offset = 0, $rowCount = 0, $sort = NULL,
6742     $count = FALSE, $includeContactIds = FALSE,
6743     $sortByChar = FALSE, $groupContacts = FALSE,
6744     $additionalWhereClause = NULL, $sortOrder = NULL,
6745     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6746
6747     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6748
6749     if ($sortByChar) {
6750       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6751       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6752     }
6753
6754     if ($skipOrderAndLimit) {
6755       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6756       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6757     }
6758     else {
6759       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6760     }
6761     return $query;
6762   }
6763
6764   /**
6765    * Get the component parts of the search query as an array.
6766    *
6767    * @param int $offset
6768    *   The offset for the query.
6769    * @param int $rowCount
6770    *   The number of rows to return.
6771    * @param string|CRM_Utils_Sort $sort
6772    *   The order by string.
6773    * @param bool $count
6774    *   Is this a count only query ?.
6775    * @param bool $includeContactIds
6776    *   Should we include contact ids?.
6777    * @param bool $sortByChar
6778    *   If true returns the distinct array of first characters for search results.
6779    * @param bool $groupContacts
6780    *   If true, return only the contact ids.
6781    * @param string $additionalWhereClause
6782    *   If the caller wants to further restrict the search (used for components).
6783    * @param null $sortOrder
6784    * @param string $additionalFromClause
6785    *   Should be clause with proper joins, effective to reduce where clause load.
6786    *
6787    * @return array
6788    * @throws \CRM_Core_Exception
6789    */
6790   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6791     $count = FALSE, $includeContactIds = FALSE,
6792     $sortByChar = FALSE, $groupContacts = FALSE,
6793     $additionalWhereClause = NULL, $sortOrder = NULL,
6794     $additionalFromClause = NULL) {
6795     if ($includeContactIds) {
6796       $this->_includeContactIds = TRUE;
6797       $this->_whereClause = $this->whereClause();
6798     }
6799     $onlyDeleted = in_array([
6800       'deleted_contacts',
6801       '=',
6802       '1',
6803       '0',
6804       '0',
6805     ], $this->_params);
6806
6807     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6808     // and that contact happens to be deleted, set $onlyDeleted to true
6809     foreach ($this->_params as $values) {
6810       $name = $values[0] ?? NULL;
6811       $op = $values[1] ?? NULL;
6812       $value = $values[2] ?? NULL;
6813       if ($name === 'contact_id' and $op === '=') {
6814         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6815           $onlyDeleted = TRUE;
6816         }
6817         break;
6818       }
6819     }
6820
6821     // building the query string
6822     $groupBy = $groupByCols = NULL;
6823     if (!$count) {
6824       if (isset($this->_groupByComponentClause)) {
6825         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6826         $groupByCols = explode(', ', $groupByCols);
6827       }
6828       elseif ($this->_useGroupBy) {
6829         $groupByCols = ['contact_a.id'];
6830       }
6831     }
6832     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6833       $groupByCols = ['civicrm_activity.id'];
6834     }
6835     if (!empty($groupByCols)) {
6836       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6837     }
6838
6839     $order = $orderBy = '';
6840     if (!$count) {
6841       if (!$sortByChar) {
6842         $order = $this->prepareOrderBy($sort, $sortOrder);
6843       }
6844     }
6845     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6846     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6847     //     clause. This will impact the search query output.
6848     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6849
6850     if ($disableFullGroupByMode) {
6851       CRM_Core_DAO::disableFullGroupByMode();
6852     }
6853
6854     // CRM-15231
6855     $this->_sort = $sort;
6856
6857     //CRM-15967
6858     $this->includePseudoFieldsJoin($sort);
6859
6860     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6861
6862     if ($additionalWhereClause) {
6863       $where = $where . ' AND ' . $additionalWhereClause;
6864     }
6865
6866     //additional from clause should be w/ proper joins.
6867     if ($additionalFromClause) {
6868       $from .= "\n" . $additionalFromClause;
6869     }
6870
6871     // if we are doing a transform, do it here
6872     // use the $from, $where and $having to get the contact ID
6873     if ($this->_displayRelationshipType) {
6874       $this->filterRelatedContacts($from, $where, $having);
6875     }
6876     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6877
6878     return [
6879       'select' => $select,
6880       'from' => $from,
6881       'where' => $where,
6882       'order_by' => $order,
6883       'group_by' => $groupBy,
6884       'having' => $having,
6885       'limit' => $limit,
6886     ];
6887   }
6888
6889   /**
6890    * Get the metadata for a given field.
6891    *
6892    * @param string $fieldName
6893    *
6894    * @return array
6895    */
6896   protected function getMetadataForField($fieldName) {
6897     if ($fieldName === 'contact_a.id') {
6898       // This seems to be the only anomaly.
6899       $fieldName = 'id';
6900     }
6901     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6902     $field = $this->_fields[$fieldName] ?? $pseudoField;
6903     $field = array_merge($field, $pseudoField);
6904     if (!empty($field) && empty($field['name'])) {
6905       // standardising field formatting here - over time we can phase out variants.
6906       // all paths using this currently unit tested
6907       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6908     }
6909     return $field;
6910   }
6911
6912   /**
6913    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6914    *
6915    * @param string $fieldName
6916    *
6917    * @return array
6918    */
6919   public function getMetadataForRealField($fieldName) {
6920     $field = $this->getMetadataForField($fieldName);
6921     if (!empty($field['is_pseudofield_for'])) {
6922       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6923       $field['pseudofield_name'] = $fieldName;
6924     }
6925     elseif (!empty($field['pseudoconstant'])) {
6926       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6927         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6928         if (empty($field['table_name'])) {
6929           if (!empty($field['where'])) {
6930             $field['table_name'] = explode('.', $field['where'])[0];
6931           }
6932           else {
6933             $field['table_name'] = 'civicrm_contact';
6934           }
6935         }
6936       }
6937     }
6938     return $field;
6939   }
6940
6941   /**
6942    * Get the field datatype, using the type in the database rather than the pseudofield, if a pseudofield.
6943    *
6944    * @param string $fieldName
6945    *
6946    * @return string
6947    */
6948   public function getDataTypeForRealField($fieldName) {
6949     return CRM_Utils_Type::typeToString($this->getMetadataForRealField($fieldName)['type']);
6950   }
6951
6952   /**
6953    * If we have a field that is better rendered via the pseudoconstant handled them here.
6954    *
6955    * Rather than joining in the additional table we render the option value on output.
6956    *
6957    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6958    * carefully with test coverage but aim to extend.
6959    *
6960    * @param string $name
6961    */
6962   protected function addPseudoconstantFieldToSelect($name) {
6963     $field = $this->getMetadataForRealField($name);
6964     $realFieldName = $field['name'];
6965     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6966     if ($pseudoFieldName) {
6967       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6968       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6969         'pseudoField' => $pseudoFieldName,
6970         'idCol' => $realFieldName,
6971         'field_name' => $field['name'],
6972         'bao' => $field['bao'],
6973         'pseudoconstant' => $field['pseudoconstant'],
6974       ];
6975     }
6976
6977     $this->_tables[$field['table_name']] = 1;
6978     $this->_element[$realFieldName] = 1;
6979     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6980   }
6981
6982   /**
6983    * Is this pseudofield a foreign key constraint.
6984    *
6985    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
6986    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
6987    * array which is done for contributions which are mostly handled as pseudoconstants.
6988    *
6989    * @param $fieldSpec
6990    *
6991    * @return bool
6992    */
6993   protected function isPseudoFieldAnFK($fieldSpec) {
6994     if (empty($fieldSpec['FKClassName'])
6995       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
6996       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
6997       return FALSE;
6998     }
6999     return TRUE;
7000   }
7001
7002   /**
7003    * Is the field a relative date field.
7004    *
7005    * @param string $fieldName
7006    *
7007    * @return bool
7008    */
7009   protected function isARelativeDateField($fieldName) {
7010     if (substr($fieldName, -9, 9) !== '_relative') {
7011       return FALSE;
7012     }
7013     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
7014     return isset($this->_fields[$realField]);
7015   }
7016
7017   /**
7018    * Get the specifications for the field, if available.
7019    *
7020    * @param string $fieldName
7021    *   Fieldname as displayed on the form.
7022    *
7023    * @return array
7024    */
7025   public function getFieldSpec($fieldName) {
7026     if (isset($this->_fields[$fieldName])) {
7027       $fieldSpec = $this->_fields[$fieldName];
7028       if (!empty($fieldSpec['is_pseudofield_for'])) {
7029         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
7030       }
7031       return $fieldSpec;
7032     }
7033     $lowFieldName = str_replace('_low', '', $fieldName);
7034     if (isset($this->_fields[$lowFieldName])) {
7035       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
7036     }
7037     $highFieldName = str_replace('_high', '', $fieldName);
7038     if (isset($this->_fields[$highFieldName])) {
7039       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
7040     }
7041     return [];
7042   }
7043
7044   public function buildWhereForDate() {
7045
7046   }
7047
7048   /**
7049    * Is the field a relative date field.
7050    *
7051    * @param string $fieldName
7052    *
7053    * @return bool
7054    */
7055   protected function isADateRangeField($fieldName) {
7056     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7057       return FALSE;
7058     }
7059     return !empty($this->getFieldSpec($fieldName));
7060   }
7061
7062   /**
7063    * @param $values
7064    */
7065   protected function buildRelativeDateQuery(&$values) {
7066     $value = $values[2] ?? NULL;
7067     if (empty($value)) {
7068       return;
7069     }
7070     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7071     $fieldSpec = $this->_fields[$fieldName];
7072     $tableName = $fieldSpec['table_name'];
7073     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7074     $grouping = $values[3] ?? NULL;
7075     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7076     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7077     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7078
7079     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7080     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7081     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7082
7083     $where = $fieldSpec['where'];
7084     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7085       // Special handling for contact table as it has a known alias in advanced search.
7086       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7087       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7088     }
7089
7090     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7091     if ($fieldName === 'relation_active_period_date') {
7092       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7093       // Not sure the  'right' fix.
7094       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7095       return;
7096     }
7097
7098     if (empty($dates[0])) {
7099       // ie. no start date we only have end date
7100       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7101     }
7102     elseif (empty($dates[1])) {
7103
7104       // ie. no end date we only have start date
7105       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7106     }
7107     else {
7108       // we have start and end dates.
7109       if ($secondWhere !== $where) {
7110         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7111       }
7112       else {
7113         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7114       }
7115     }
7116   }
7117
7118   /**
7119    * Build the query for a date field if it is a _high or _low field.
7120    *
7121    * @param $values
7122    *
7123    * @return bool
7124    * @throws \CRM_Core_Exception
7125    */
7126   public function buildDateRangeQuery($values) {
7127     if ($this->isADateRangeField($values[0])) {
7128       $fieldSpec = $this->getFieldSpec($values[0]);
7129       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7130       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7131       return TRUE;
7132     }
7133     return FALSE;
7134   }
7135
7136   /**
7137    * Add the address table into the query.
7138    *
7139    * @param string $tableKey
7140    * @param string $joinCondition
7141    *
7142    * @return array
7143    *   - alias name
7144    *   - address join.
7145    */
7146   protected function addAddressTable($tableKey, $joinCondition) {
7147     $tName = "$tableKey-address";
7148     $aName = "`$tableKey-address`";
7149     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7150     $this->_element["{$tName}_id"] = 1;
7151     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7152     $this->_tables[$tName] = $addressJoin;
7153
7154     return [
7155       $aName,
7156       $addressJoin,
7157     ];
7158   }
7159
7160   /**
7161    * Get the clause for group status.
7162    *
7163    * @param int $grouping
7164    *
7165    * @return string
7166    */
7167   protected function getGroupStatusClause($grouping) {
7168     $statuses = $this->getSelectedGroupStatuses($grouping);
7169     return "status IN (" . implode(', ', $statuses) . ")";
7170   }
7171
7172   /**
7173    * Get an array of the statuses that have been selected.
7174    *
7175    * @param string $grouping
7176    *
7177    * @return array
7178    *
7179    * @throws \CRM_Core_Exception
7180    */
7181   protected function getSelectedGroupStatuses($grouping) {
7182     $statuses = [];
7183     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7184     if ($gcsValues &&
7185       is_array($gcsValues[2])
7186     ) {
7187       foreach ($gcsValues[2] as $k => $v) {
7188         if ($v) {
7189           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7190         }
7191       }
7192     }
7193     else {
7194       $statuses[] = "'Added'";
7195     }
7196     return $statuses;
7197   }
7198
7199   /**
7200    * Get the qill value for the field.
7201    *
7202    * @param string $name
7203    * @param array|int|string $value
7204    * @param string $op
7205    * @param array $fieldSpec
7206    * @param string $labelOverride
7207    *   Label override, if required.
7208    *
7209    * @return string
7210    */
7211   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7212     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7213     return (string) ts("%1 %2 %3", [
7214       1 => $labelOverride ?? $fieldSpec['title'],
7215       2 => $qillop,
7216       3 => $qillVal,
7217     ]);
7218   }
7219
7220   /**
7221    * Where handling for any field with adequately defined metadata.
7222    *
7223    * @param array $fieldSpec
7224    * @param string $name
7225    * @param string|array|int $value
7226    * @param string $op
7227    * @param string|int $grouping
7228    *
7229    * @throws \CRM_Core_Exception
7230    */
7231   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7232     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7233     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7234     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7235       $this->_tables[$fieldSpec['table_name']] = 1;
7236     }
7237     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7238       $this->_whereTables[$fieldSpec['table_name']] = 1;
7239     }
7240   }
7241
7242   /**
7243    * Get the qill for the relative date range.
7244    *
7245    * @param string|null $from
7246    * @param string|null $to
7247    * @param string $fieldTitle
7248    * @param string $relativeRange
7249    *
7250    * @return string
7251    */
7252   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7253     if (!$from) {
7254       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7255     }
7256     if (!$to) {
7257       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7258     }
7259     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7260       CRM_Utils_Date::customFormat($from),
7261       CRM_Utils_Date::customFormat($to),
7262     ]) . ')';
7263   }
7264
7265 }