CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * The cache to translate the option values into labels.
 218    *
 219    * @var array
 220    */
 221   public $_options;
 222
 223   /**
 224    * Are we in search mode.
 225    *
 226    * @var bool
 227    */
 228   public $_search = TRUE;
 229
 230   /**
 231    * Should we skip permission checking.
 232    *
 233    * @var bool
 234    */
 235   public $_skipPermission = FALSE;
 236
 237   /**
 238    * Should we skip adding of delete clause.
 239    *
 240    * @var bool
 241    */
 242   public $_skipDeleteClause = FALSE;
 243
 244   /**
 245    * Are we in strict mode (use equality over LIKE)
 246    *
 247    * @var bool
 248    */
 249   public $_strict = FALSE;
 250
 251   /**
 252    * What operator to use to group the clauses.
 253    *
 254    * @var string
 255    */
 256   public $_operator = 'AND';
 257
 258   public $_mode = 1;
 259
 260   /**
 261    * Should we only search on primary location.
 262    *
 263    * @var bool
 264    */
 265   public $_primaryLocation = TRUE;
 266
 267   /**
 268    * Are contact ids part of the query.
 269    *
 270    * @var bool
 271    */
 272   public $_includeContactIds = FALSE;
 273
 274   /**
 275    * Should we use the smart group cache.
 276    *
 277    * @var bool
 278    */
 279   public $_smartGroupCache = TRUE;
 280
 281   /**
 282    * Should we display contacts with a specific relationship type.
 283    *
 284    * @var string
 285    */
 286   public $_displayRelationshipType;
 287
 288   /**
 289    * Reference to the query object for custom values.
 290    *
 291    * @var Object
 292    */
 293   public $_customQuery;
 294
 295   /**
 296    * Should we enable the distinct clause, used if we are including
 297    * more than one group
 298    *
 299    * @var bool
 300    */
 301   public $_useDistinct = FALSE;
 302
 303   /**
 304    * Should we just display one contact record
 305    * @var bool
 306    */
 307   public $_useGroupBy = FALSE;
 308
 309   /**
 310    * The relationship type direction
 311    *
 312    * @var array
 313    */
 314   public static $_relType;
 315
 316   /**
 317    * The activity role
 318    *
 319    * @var array
 320    */
 321   public static $_activityRole;
 322
 323   /**
 324    * Consider the component activity type
 325    * during activity search.
 326    *
 327    * @var array
 328    */
 329   public static $_considerCompActivities;
 330
 331   /**
 332    * Consider with contact activities only,
 333    * during activity search.
 334    *
 335    * @var array
 336    */
 337   public static $_withContactActivitiesOnly;
 338
 339   /**
 340    * Use distinct component clause for component searches
 341    *
 342    * @var string
 343    */
 344   public $_distinctComponentClause;
 345
 346   public $_rowCountClause;
 347
 348   /**
 349    * Use groupBy component clause for component searches
 350    *
 351    * @var string
 352    */
 353   public $_groupByComponentClause;
 354
 355   /**
 356    * Track open panes, useful in advance search
 357    *
 358    * @var array
 359    */
 360   public static $_openedPanes = [];
 361
 362   /**
 363    * For search builder - which custom fields are location-dependent
 364    * @var array
 365    */
 366   public $_locationSpecificCustomFields = [];
 367
 368   /**
 369    * The tables which have a dependency on location and/or address
 370    *
 371    * @var array
 372    */
 373   public static $_dependencies = [
 374     'civicrm_state_province' => 1,
 375     'civicrm_country' => 1,
 376     'civicrm_county' => 1,
 377     'civicrm_address' => 1,
 378     'civicrm_location_type' => 1,
 379   ];
 380
 381   /**
 382    * List of location specific fields.
 383    * @var array
 384    */
 385   public static $_locationSpecificFields = [
 386     'street_address',
 387     'street_number',
 388     'street_name',
 389     'street_unit',
 390     'supplemental_address_1',
 391     'supplemental_address_2',
 392     'supplemental_address_3',
 393     'city',
 394     'postal_code',
 395     'postal_code_suffix',
 396     'geo_code_1',
 397     'geo_code_2',
 398     'state_province',
 399     'country',
 400     'county',
 401     'phone',
 402     'email',
 403     'im',
 404     'address_name',
 405     'master_id',
 406   ];
 407
 408   /**
 409    * Remember if we handle either end of a number or date range
 410    * so we can skip the other
 411    * @var array
 412    */
 413   protected $_rangeCache = [];
 414   /**
 415    * Set to true when $this->relationship is run to avoid adding twice.
 416    *
 417    * @var bool
 418    */
 419   protected $_relationshipValuesAdded = FALSE;
 420
 421   /**
 422    * Set to the name of the temp table if one has been created.
 423    *
 424    * @var string
 425    */
 426   public static $_relationshipTempTable;
 427
 428   public $_pseudoConstantsSelect = [];
 429
 430   public $_groupUniqueKey;
 431   public $_groupKeys = [];
 432
 433   /**
 434    * Class constructor which also does all the work.
 435    *
 436    * @param array $params
 437    * @param array $returnProperties
 438    * @param array $fields
 439    * @param bool $includeContactIds
 440    * @param bool $strict
 441    * @param bool|int $mode - mode the search is operating on
 442    *
 443    * @param bool $skipPermission
 444    * @param bool $searchDescendentGroups
 445    * @param bool $smartGroupCache
 446    * @param null $displayRelationshipType
 447    * @param string $operator
 448    * @param string $apiEntity
 449    * @param bool|null $primaryLocationOnly
 450    *
 451    * @throws \CRM_Core_Exception
 452    */
 453   public function __construct(
 454     $params = NULL, $returnProperties = NULL, $fields = NULL,
 455     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 456     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 457     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 458     $operator = 'AND',
 459     $apiEntity = NULL,
 460     $primaryLocationOnly = NULL
 461   ) {
 462     if ($primaryLocationOnly === NULL) {
 463       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 464     }
 465     $this->_primaryLocation = $primaryLocationOnly;
 466     $this->_params = &$params;
 467     if ($this->_params == NULL) {
 468       $this->_params = [];
 469     }
 470
 471     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 472       $this->_returnProperties = [];
 473     }
 474     elseif (empty($returnProperties)) {
 475       $this->_returnProperties = self::defaultReturnProperties($mode);
 476     }
 477     else {
 478       $this->_returnProperties = &$returnProperties;
 479     }
 480
 481     $this->_includeContactIds = $includeContactIds;
 482     $this->_strict = $strict;
 483     $this->_mode = $mode;
 484     $this->_skipPermission = $skipPermission;
 485     $this->_smartGroupCache = $smartGroupCache;
 486     $this->_displayRelationshipType = $displayRelationshipType;
 487     $this->setOperator($operator);
 488
 489     if ($fields) {
 490       $this->_fields = &$fields;
 491       $this->_search = FALSE;
 492       $this->_skipPermission = TRUE;
 493     }
 494     else {
 495       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 496       //  The legacy hacked fields will output as a string rather than their underlying type.
 497       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 498         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 499       }
 500       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 501       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 502       // No good option other than hard-coding metadata for this 'special' field in.
 503       $relationFields['relation_active_period_date'] = [
 504         'name' => 'relation_active_period_date',
 505         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 506         'title' => ts('Active Period'),
 507         'table_name' => 'civicrm_relationship',
 508         'where' => 'civicrm_relationship.start_date',
 509         'where_end' => 'civicrm_relationship.end_date',
 510         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 511       ];
 512       $this->_fields = array_merge($relationFields, $this->_fields);
 513
 514       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 515       unset($fields['note']);
 516       $this->_fields = array_merge($this->_fields, $fields);
 517
 518       // add activity fields
 519       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 520       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields('Case'));
 521       // Add hack as no unique name is defined for the field but the search form is in denial.
 522       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 523
 524       // add any fields provided by hook implementers
 525       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 526       $this->_fields = array_merge($this->_fields, $extFields);
 527     }
 528
 529     // basically do all the work once, and then reuse it
 530     $this->initialize($apiEntity);
 531   }
 532
 533   /**
 534    * Function which actually does all the work for the constructor.
 535    *
 536    * @param string $apiEntity
 537    *   The api entity being called.
 538    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 539    *
 540    * @throws \CRM_Core_Exception
 541    */
 542   public function initialize($apiEntity = NULL) {
 543     $this->_select = [];
 544     $this->_element = [];
 545     $this->_tables = [];
 546     $this->_whereTables = [];
 547     $this->_where = [];
 548     $this->_qill = [];
 549     $this->_options = [];
 550     $this->_cfIDs = [];
 551     $this->_paramLookup = [];
 552     $this->_having = [];
 553
 554     $this->_customQuery = NULL;
 555
 556     // reset cached static variables - CRM-5803
 557     self::$_activityRole = NULL;
 558     self::$_considerCompActivities = NULL;
 559     self::$_withContactActivitiesOnly = NULL;
 560
 561     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 562     $this->_element['contact_id'] = 1;
 563     $this->_tables['civicrm_contact'] = 1;
 564
 565     if (!empty($this->_params)) {
 566       $this->buildParamsLookup();
 567     }
 568
 569     $this->_whereTables = $this->_tables;
 570
 571     $this->selectClause($apiEntity);
 572     if (!empty($this->_cfIDs)) {
 573       // @todo This function is the select function but instead of running 'select' it
 574       // is running the whole query.
 575       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 576       $this->_customQuery->query();
 577       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 578       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 579       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 580       $this->_options = $this->_customQuery->_options;
 581     }
 582     $isForcePrimaryOnly = !empty($apiEntity);
 583     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 584     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 585       $component = 'contribution';
 586     }
 587     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 588       $component = 'membership';
 589     }
 590     if (isset($component) && !$this->_skipPermission) {
 591       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 592       CRM_Financial_BAO_FinancialType::buildPermissionedClause($this->_whereClause, $component);
 593     }
 594
 595     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 596     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 597
 598     $this->openedSearchPanes(TRUE);
 599   }
 600
 601   /**
 602    * Function for same purpose as convertFormValues.
 603    *
 604    * Like convert form values this function exists to pre-Process parameters from the form.
 605    *
 606    * It is unclear why they are different functions & likely relates to advances search
 607    * versus search builder.
 608    *
 609    * The direction we are going is having the form convert values to a standardised format &
 610    * moving away from weird & wonderful where clause switches.
 611    *
 612    * Fix and handle contact deletion nicely.
 613    *
 614    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 615    *
 616    * @see https://issues.civicrm.org/jira/browse/CRM-11971
 617    */
 618   public function buildParamsLookup() {
 619     $trashParamExists = FALSE;
 620     $paramByGroup = [];
 621     foreach ($this->_params as $k => $param) {
 622       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 623         $trashParamExists = TRUE;
 624       }
 625       if (!empty($param[3])) {
 626         $paramByGroup[$param[3]][$k] = $param;
 627       }
 628     }
 629
 630     if ($trashParamExists) {
 631       $this->_skipDeleteClause = TRUE;
 632
 633       //cycle through group sets and explicitly add trash param if not set
 634       foreach ($paramByGroup as $setID => $set) {
 635         if (
 636           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 637           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 638         ) {
 639           $this->_params[] = [
 640             'contact_is_deleted',
 641             '=',
 642             '0',
 643             $setID,
 644             '0',
 645           ];
 646         }
 647       }
 648     }
 649
 650     foreach ($this->_params as $value) {
 651       if (empty($value[0])) {
 652         continue;
 653       }
 654       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 655       if ($cfID) {
 656         if (!array_key_exists($cfID, $this->_cfIDs)) {
 657           $this->_cfIDs[$cfID] = [];
 658         }
 659         // Set wildcard value based on "and/or" selection
 660         foreach ($this->_params as $key => $param) {
 661           if ($param[0] == $value[0] . '_operator') {
 662             $value[4] = $param[2] == 'or';
 663             break;
 664           }
 665         }
 666         $this->_cfIDs[$cfID][] = $value;
 667       }
 668
 669       if (!array_key_exists($value[0], $this->_paramLookup)) {
 670         $this->_paramLookup[$value[0]] = [];
 671       }
 672       if ($value[0] !== 'group') {
 673         // Just trying to unravel how group interacts here! This whole function is weird.
 674         $this->_paramLookup[$value[0]][] = $value;
 675       }
 676     }
 677   }
 678
 679   /**
 680    * Some composite fields do not appear in the fields array hack to make them part of the query.
 681    *
 682    * @param $apiEntity
 683    *   The api entity being called.
 684    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 685    */
 686   public function addSpecialFields($apiEntity) {
 687     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 688     // if get called via Contact.get API having address_id as return parameter
 689     if ($apiEntity === 'Contact') {
 690       $special[] = 'address_id';
 691     }
 692     foreach ($special as $name) {
 693       if (!empty($this->_returnProperties[$name])) {
 694         if ($name === 'address_id') {
 695           $this->_tables['civicrm_address'] = 1;
 696           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 697           $this->_element['address_id'] = 1;
 698         }
 699         else {
 700           $this->_select[$name] = "contact_a.{$name} as $name";
 701           $this->_element[$name] = 1;
 702         }
 703       }
 704     }
 705   }
 706
 707   /**
 708    * Given a list of conditions in params and a list of desired
 709    * return Properties generate the required select and from
 710    * clauses. Note that since the where clause introduces new
 711    * tables, the initial attempt also retrieves all variables used
 712    * in the params list
 713    *
 714    * @param string $apiEntity
 715    *   The api entity being called.
 716    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 717    */
 718   public function selectClause($apiEntity = NULL) {
 719
 720     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 721     // param ($apiEntity) instead - presumably an oversight & 2 because
 722     // contact is not implemented as a component.
 723     $this->addSpecialFields($apiEntity);
 724
 725     foreach ($this->_fields as $name => $field) {
 726       // skip component fields
 727       // there are done by the alter query below
 728       // and need not be done on every field
 729       // @todo remove these & handle using metadata - only obscure fields
 730       // that are hack-added should need to be excluded from the main loop.
 731       if (
 732         (substr($name, 0, 12) === 'participant_') ||
 733         (substr($name, 0, 7) === 'pledge_') ||
 734         (substr($name, 0, 5) === 'case_')
 735       ) {
 736         continue;
 737       }
 738
 739       // redirect to activity select clause
 740       if (
 741         (substr($name, 0, 9) === 'activity_') ||
 742         ($name === 'parent_id')
 743       ) {
 744         CRM_Activity_BAO_Query::select($this);
 745       }
 746
 747       // if this is a hierarchical name, we ignore it
 748       $names = explode('-', $name);
 749       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 750         continue;
 751       }
 752
 753       // make an exception for special cases, to add the field in select clause
 754       $makeException = FALSE;
 755
 756       //special handling for groups/tags
 757       if (in_array($name, ['groups', 'tags', 'notes'])
 758         && isset($this->_returnProperties[substr($name, 0, -1)])
 759       ) {
 760         // @todo instead of setting make exception to get us into
 761         // an if clause that has handling for these fields buried with in it
 762         // move the handling to here.
 763         $makeException = TRUE;
 764       }
 765
 766       // since note has 3 different options we need special handling
 767       // note / note_subject / note_body
 768       if ($name === 'notes') {
 769         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 770           if (isset($this->_returnProperties[$noteField])) {
 771             $makeException = TRUE;
 772             break;
 773           }
 774         }
 775       }
 776
 777       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 778       if (
 779         !empty($this->_paramLookup[$name])
 780         || !empty($this->_returnProperties[$name])
 781         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 782         || $makeException
 783       ) {
 784         if ($cfID) {
 785           // add to cfIDs array if not present
 786           if (!array_key_exists($cfID, $this->_cfIDs)) {
 787             $this->_cfIDs[$cfID] = [];
 788           }
 789         }
 790         elseif (isset($field['where'])) {
 791           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 792           if (isset($tableName)) {
 793             if (!empty(self::$_dependencies[$tableName])) {
 794               $this->_tables['civicrm_address'] = 1;
 795               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 796               $this->_element['address_id'] = 1;
 797             }
 798
 799             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 800               $tableName === 'postal_greeting' || $tableName === 'addressee'
 801             ) {
 802               if ($tableName === 'im_provider') {
 803                 CRM_Core_OptionValue::select($this);
 804               }
 805
 806               if (in_array($tableName,
 807                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 808                 $this->_element["{$name}_id"] = 1;
 809                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 810                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 811                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 812                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 813
 814                 if ($tableName === 'email_greeting') {
 815                   // @todo bad join.
 816                   $this->_pseudoConstantsSelect[$name]['join']
 817                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 818                   $this->_pseudoConstantsSelect[$name]['join'] .=
 819                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 820                 }
 821                 elseif ($tableName === 'postal_greeting') {
 822                   // @todo bad join.
 823                   $this->_pseudoConstantsSelect[$name]['join']
 824                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 825                   $this->_pseudoConstantsSelect[$name]['join'] .=
 826                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 827                 }
 828                 elseif ($tableName == 'addressee') {
 829                   // @todo bad join.
 830                   $this->_pseudoConstantsSelect[$name]['join']
 831                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 832                   $this->_pseudoConstantsSelect[$name]['join'] .=
 833                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 834                 }
 835                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 836
 837                 //get display
 838                 $greetField = "{$name}_display";
 839                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 840                 $this->_element[$greetField] = 1;
 841                 //get custom
 842                 $greetField = "{$name}_custom";
 843                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 844                 $this->_element[$greetField] = 1;
 845               }
 846             }
 847             else {
 848               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 849                 $this->_tables[$tableName] = 1;
 850               }
 851
 852               // also get the id of the tableName
 853               $tName = substr($tableName, 8);
 854               if (in_array($tName, ['country', 'state_province', 'county'])) {
 855                 if ($tName == 'state_province') {
 856                   $this->_pseudoConstantsSelect['state_province_name'] = [
 857                     'pseudoField' => "{$tName}",
 858                     'idCol' => "{$tName}_id",
 859                     'bao' => 'CRM_Core_BAO_Address',
 860                     'table' => "civicrm_{$tName}",
 861                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 862                   ];
 863
 864                   $this->_pseudoConstantsSelect[$tName] = [
 865                     'pseudoField' => 'state_province_abbreviation',
 866                     'idCol' => "{$tName}_id",
 867                     'table' => "civicrm_{$tName}",
 868                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 869                   ];
 870                 }
 871                 else {
 872                   $this->_pseudoConstantsSelect[$name] = [
 873                     'pseudoField' => "{$tName}_id",
 874                     'idCol' => "{$tName}_id",
 875                     'bao' => 'CRM_Core_BAO_Address',
 876                     'table' => "civicrm_{$tName}",
 877                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 878                   ];
 879                 }
 880
 881                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 882                 $this->_element["{$tName}_id"] = 1;
 883               }
 884               elseif ($tName != 'contact') {
 885                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 886                 $this->_element["{$tName}_id"] = 1;
 887               }
 888
 889               //special case for phone
 890               if ($name == 'phone') {
 891                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 892                 $this->_element['phone_type_id'] = 1;
 893               }
 894
 895               // if IM then select provider_id also
 896               // to get "IM Service Provider" in a file to be exported, CRM-3140
 897               if ($name == 'im') {
 898                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 899                 $this->_element['provider_id'] = 1;
 900               }
 901
 902               if ($tName == 'contact' && $fieldName == 'organization_name') {
 903                 // special case, when current employer is set for Individual contact
 904                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 905               }
 906               elseif ($tName == 'contact' && $fieldName === 'id') {
 907                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 908               }
 909               elseif (in_array($tName, ['country', 'county'])) {
 910                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 911                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 912               }
 913               elseif ($tName == 'state_province') {
 914                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 915                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 916               }
 917               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 918                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 919                   $this->_select[$name] = "{$field['where']} as `$name`";
 920                 }
 921               }
 922               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 923                 $this->addPseudoconstantFieldToSelect($name);
 924               }
 925               else {
 926                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 927               }
 928               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 929                 $this->_element[$name] = 1;
 930               }
 931             }
 932           }
 933         }
 934         elseif ($name === 'tags') {
 935           //@todo move this handling outside the big IF & ditch $makeException
 936           $this->_useGroupBy = TRUE;
 937           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 938           $this->_element[$name] = 1;
 939           $this->_tables['civicrm_tag'] = 1;
 940           $this->_tables['civicrm_entity_tag'] = 1;
 941         }
 942         elseif ($name === 'groups') {
 943           //@todo move this handling outside the big IF & ditch $makeException
 944           $this->_useGroupBy = TRUE;
 945           // Duplicates will be created here but better to sort them out in php land.
 946           $this->_select[$name] = "
 947             CONCAT_WS(',',
 948             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 949             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 950           )
 951           as `groups`";
 952           $this->_element[$name] = 1;
 953           $this->_tables['civicrm_group_contact'] = 1;
 954           $this->_tables['civicrm_group_contact_cache'] = 1;
 955           $this->_pseudoConstantsSelect["{$name}"] = [
 956             'pseudoField' => "groups",
 957             'idCol' => 'groups',
 958           ];
 959         }
 960         elseif ($name === 'notes') {
 961           //@todo move this handling outside the big IF & ditch $makeException
 962           // if note field is subject then return subject else body of the note
 963           $noteColumn = 'note';
 964           if (isset($noteField) && $noteField === 'note_subject') {
 965             $noteColumn = 'subject';
 966           }
 967
 968           $this->_useGroupBy = TRUE;
 969           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 970           $this->_element[$name] = 1;
 971           $this->_tables['civicrm_note'] = 1;
 972         }
 973         elseif ($name === 'current_employer') {
 974           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 975           $this->_element[$name] = 1;
 976         }
 977       }
 978
 979       if ($cfID && !empty($field['is_search_range'])) {
 980         // this is a custom field with range search enabled, so we better check for two/from values
 981         if (!empty($this->_paramLookup[$name . '_from'])) {
 982           if (!array_key_exists($cfID, $this->_cfIDs)) {
 983             $this->_cfIDs[$cfID] = [];
 984           }
 985           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 986             // search in the cdID array for the same grouping
 987             $fnd = FALSE;
 988             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 989               if ($c[3] == $p[3]) {
 990                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 991                 $fnd = TRUE;
 992               }
 993             }
 994             if (!$fnd) {
 995               $p[2] = ['from' => $p[2]];
 996               $this->_cfIDs[$cfID][] = $p;
 997             }
 998           }
 999         }
1000         if (!empty($this->_paramLookup[$name . '_to'])) {
1001           if (!array_key_exists($cfID, $this->_cfIDs)) {
1002             $this->_cfIDs[$cfID] = [];
1003           }
1004           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
1005             // search in the cdID array for the same grouping
1006             $fnd = FALSE;
1007             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
1008               if ($c[4] == $p[4]) {
1009                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1010                 $fnd = TRUE;
1011               }
1012             }
1013             if (!$fnd) {
1014               $p[2] = ['to' => $p[2]];
1015               $this->_cfIDs[$cfID][] = $p;
1016             }
1017           }
1018         }
1019       }
1020     }
1021
1022     // add location as hierarchical elements
1023     $this->addHierarchicalElements();
1024
1025     // add multiple field like website
1026     $this->addMultipleElements();
1027
1028     //fix for CRM-951
1029     CRM_Core_Component::alterQuery($this, 'select');
1030
1031     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1032   }
1033
1034   /**
1035    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1036    */
1037   public function addHierarchicalElements() {
1038     if (empty($this->_returnProperties['location'])) {
1039       return;
1040     }
1041     if (!is_array($this->_returnProperties['location'])) {
1042       return;
1043     }
1044
1045     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1046     $processed = [];
1047     $index = 0;
1048
1049     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1050     $addressCustomFieldIds = [];
1051
1052     foreach ($this->_returnProperties['location'] as $name => $elements) {
1053       $lCond = self::getPrimaryCondition($name);
1054       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1055
1056       if (!$lCond) {
1057         if ($locationTypeId === FALSE) {
1058           continue;
1059         }
1060         $lCond = "location_type_id = $locationTypeId";
1061         $this->_useDistinct = TRUE;
1062
1063         //commented for CRM-3256
1064         $this->_useGroupBy = TRUE;
1065       }
1066
1067       $name = str_replace(' ', '_', $name);
1068       $tName = "$name-location_type";
1069       $ltName = "`$name-location_type`";
1070       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1071       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1072       $this->_element["{$tName}_id"] = 1;
1073       $this->_element["{$tName}"] = 1;
1074
1075       $locationTypeName = $tName;
1076       $locationTypeJoin = [];
1077
1078       $addWhereCount = 0;
1079       foreach ($elements as $elementFullName => $dontCare) {
1080         $index++;
1081         $elementName = $elementCmpName = $elementFullName;
1082
1083         if (substr($elementCmpName, 0, 5) == 'phone') {
1084           $elementCmpName = 'phone';
1085         }
1086
1087         if (in_array($elementCmpName, array_keys($addressCustomFields))) {
1088           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1089             $addressCustomFieldIds[$cfID][$name] = 1;
1090           }
1091         }
1092         // add address table - doesn't matter if we do it mutliple times - it's the same data
1093         // @todo ditch the double processing of addressJoin
1094         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1095           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1096         ) {
1097           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1098           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1099           $processed[$aName] = 1;
1100         }
1101
1102         $cond = $elementType = '';
1103         if (strpos($elementName, '-') !== FALSE) {
1104           // this is either phone, email or IM
1105           list($elementName, $elementType) = explode('-', $elementName);
1106
1107           if (($elementName != 'phone') && ($elementName != 'im')) {
1108             $cond = self::getPrimaryCondition($elementType);
1109           }
1110           // CRM-13011 : If location type is primary, do not restrict search to the phone
1111           // type id - we want the primary phone, regardless of what type it is.
1112           // Otherwise, restrict to the specified phone type for the given field.
1113           if ((!$cond) && ($elementName == 'phone')) {
1114             $cond = "phone_type_id = '$elementType'";
1115           }
1116           elseif ((!$cond) && ($elementName == 'im')) {
1117             // IM service provider id, CRM-3140
1118             $cond = "provider_id = '$elementType'";
1119           }
1120           $elementType = '-' . $elementType;
1121         }
1122
1123         $field = $this->_fields[$elementName] ?? NULL;
1124         if (isset($this->_pseudoConstantsSelect[$field['name']])) {
1125           $this->_pseudoConstantsSelect[$name . '-' . $field['name']] = $this->_pseudoConstantsSelect[$field['name']];
1126         }
1127
1128         // hack for profile, add location id
1129         if (!$field) {
1130           if ($elementType &&
1131             // fix for CRM-882( to handle phone types )
1132             !is_numeric($elementType)
1133           ) {
1134             if (is_numeric($name)) {
1135               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1136             }
1137             else {
1138               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1139             }
1140           }
1141           elseif (is_numeric($name)) {
1142             //this for phone type to work
1143             if (in_array($elementName, ['phone', 'phone_ext'])) {
1144               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1145             }
1146             else {
1147               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1148             }
1149           }
1150           else {
1151             //this is for phone type to work for profile edit
1152             if (in_array($elementName, ['phone', 'phone_ext'])) {
1153               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1154             }
1155             else {
1156               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1157             }
1158           }
1159         }
1160
1161         // Check if there is a value, if so also add to where Clause
1162         $addWhere = FALSE;
1163         if ($this->_params) {
1164           $nm = $elementName;
1165           if (isset($locationTypeId)) {
1166             $nm .= "-$locationTypeId";
1167           }
1168           if (!is_numeric($elementType)) {
1169             $nm .= "$elementType";
1170           }
1171
1172           foreach ($this->_params as $id => $values) {
1173             if ((is_array($values) && $values[0] == $nm) ||
1174               (in_array($elementName, ['phone', 'im'])
1175                 && (strpos($values[0], $nm) !== FALSE)
1176               )
1177             ) {
1178               $addWhere = TRUE;
1179               $addWhereCount++;
1180               break;
1181             }
1182           }
1183         }
1184
1185         if ($field && isset($field['where'])) {
1186           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1187           $pf = substr($tableName, 8);
1188           $tName = $name . '-' . $pf . $elementType;
1189           if (isset($tableName)) {
1190             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1191               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1192             }
1193             else {
1194               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1195             }
1196
1197             $this->_element["{$tName}_id"] = 1;
1198             if (substr($tName, -15) == '-state_province') {
1199               // FIXME: hack to fix CRM-1900
1200               $a = Civi::settings()->get('address_format');
1201
1202               if (substr_count($a, 'state_province_name') > 0) {
1203                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1204                   'pseudoField' => "{$pf}_id",
1205                   'idCol' => "{$tName}_id",
1206                   'bao' => 'CRM_Core_BAO_Address',
1207                 ];
1208                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1209               }
1210               else {
1211                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1212                   'pseudoField' => 'state_province_abbreviation',
1213                   'idCol' => "{$tName}_id",
1214                 ];
1215                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1216               }
1217             }
1218             else {
1219               if (substr($elementFullName, 0, 2) == 'im') {
1220                 $provider = "{$name}-{$elementFullName}-provider_id";
1221                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1222                 $this->_element[$provider] = 1;
1223               }
1224               if ($pf == 'country' || $pf == 'county') {
1225                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1226                   'pseudoField' => "{$pf}_id",
1227                   'idCol' => "{$tName}_id",
1228                   'bao' => 'CRM_Core_BAO_Address',
1229                 ];
1230                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1231               }
1232               else {
1233                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1234               }
1235             }
1236
1237             if (in_array($pf, ['state_province', 'country', 'county'])) {
1238               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1239             }
1240             else {
1241               $this->_element["{$name}-{$elementFullName}"] = 1;
1242             }
1243
1244             if (empty($processed["`$tName`"])) {
1245               $processed["`$tName`"] = 1;
1246               $newName = $tableName . '_' . $index;
1247               switch ($tableName) {
1248                 case 'civicrm_phone':
1249                 case 'civicrm_email':
1250                 case 'civicrm_im':
1251                 case 'civicrm_openid':
1252
1253                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1254                   if ($tableName != 'civicrm_phone') {
1255                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1256                   }
1257                   elseif (is_numeric($name)) {
1258                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1259                   }
1260
1261                   // this special case to add phone type
1262                   if ($cond) {
1263                     $phoneTypeCondition = " AND `$tName`.$cond ";
1264                     //gross hack to pickup corrupted data also, CRM-7603
1265                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1266                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1267                       if (!empty($lCond)) {
1268                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1269                       }
1270                     }
1271                     $this->_tables[$tName] .= $phoneTypeCondition;
1272                   }
1273
1274                   //build locationType join
1275                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1276
1277                   if ($addWhere) {
1278                     $this->_whereTables[$tName] = $this->_tables[$tName];
1279                   }
1280                   break;
1281
1282                 case 'civicrm_state_province':
1283                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1284                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1285                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1286                   if ($addWhere) {
1287                     $this->_whereTables["{$name}-address"] = $addressJoin;
1288                   }
1289                   break;
1290
1291                 case 'civicrm_country':
1292                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1293                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1294                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1295                   if ($addWhere) {
1296                     $this->_whereTables["{$name}-address"] = $addressJoin;
1297                   }
1298                   break;
1299
1300                 case 'civicrm_county':
1301                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1302                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1303                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1304                   if ($addWhere) {
1305                     $this->_whereTables["{$name}-address"] = $addressJoin;
1306                   }
1307                   break;
1308
1309                 default:
1310                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1311                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1312                   }
1313                   if ($addWhere) {
1314                     $this->_whereTables["{$name}-address"] = $addressJoin;
1315                   }
1316                   break;
1317               }
1318             }
1319           }
1320         }
1321       }
1322
1323       // add location type  join
1324       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1325       $this->_tables[$locationTypeName] = $ltypeJoin;
1326
1327       // table should be present in $this->_whereTables,
1328       // to add its condition in location type join, CRM-3939.
1329       if ($addWhereCount) {
1330         $locClause = [];
1331         foreach ($this->_whereTables as $tableName => $clause) {
1332           if (!empty($locationTypeJoin[$tableName])) {
1333             $locClause[] = $locationTypeJoin[$tableName];
1334           }
1335         }
1336
1337         if (!empty($locClause)) {
1338           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1339         }
1340       }
1341     }
1342
1343     if (!empty($addressCustomFieldIds)) {
1344       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1345       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1346         foreach ($locTypeName as $name => $dnc) {
1347           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1348           $fieldName = "$name-custom_{$cfID}";
1349           $tName = "$name-address-custom-{$cfID}";
1350           $aName = "`$name-address-custom-{$cfID}`";
1351           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1352           $this->_element["{$tName}_id"] = 1;
1353           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1354           $this->_element[$fieldName] = 1;
1355           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1356         }
1357       }
1358     }
1359   }
1360
1361   /**
1362    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1363    */
1364   public function addMultipleElements() {
1365     if (empty($this->_returnProperties['website'])) {
1366       return;
1367     }
1368     if (!is_array($this->_returnProperties['website'])) {
1369       return;
1370     }
1371
1372     foreach ($this->_returnProperties['website'] as $key => $elements) {
1373       foreach ($elements as $elementFullName => $dontCare) {
1374         $tName = "website-{$key}-{$elementFullName}";
1375         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1376         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1377         $this->_element["{$tName}_id"] = 1;
1378         $this->_element["{$tName}"] = 1;
1379
1380         $type = "website-{$key}-website_type_id";
1381         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1382         $this->_element[$type] = 1;
1383         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1384       }
1385     }
1386   }
1387
1388   /**
1389    * Generate the query based on what type of query we need.
1390    *
1391    * @param bool $count
1392    * @param bool $sortByChar
1393    * @param bool $groupContacts
1394    * @param bool $onlyDeleted
1395    *
1396    * @return array
1397    *   sql query parts as an array
1398    */
1399   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1400     // build permission clause
1401     $this->generatePermissionClause($onlyDeleted, $count);
1402
1403     if ($count) {
1404       if (isset($this->_rowCountClause)) {
1405         $select = "SELECT {$this->_rowCountClause}";
1406       }
1407       elseif (isset($this->_distinctComponentClause)) {
1408         // we add distinct to get the right count for components
1409         // for the more complex result set, we use GROUP BY the same id
1410         // CRM-9630
1411         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1412       }
1413       else {
1414         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1415       }
1416       $from = $this->_simpleFromClause;
1417       if ($this->_useDistinct) {
1418         $this->_useGroupBy = TRUE;
1419       }
1420     }
1421     elseif ($sortByChar) {
1422       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1423       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1424       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1425       $from = $this->_simpleFromClause;
1426     }
1427     elseif ($groupContacts) {
1428       $select = 'SELECT contact_a.id as id';
1429       if ($this->_useDistinct) {
1430         $this->_useGroupBy = TRUE;
1431       }
1432       $from = $this->_simpleFromClause;
1433     }
1434     else {
1435       if (!empty($this->_paramLookup['group'])) {
1436
1437         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1438
1439         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1440           $this->_paramLookup['group'][0][1] = key($value);
1441         }
1442
1443         // Presumably the lines below come into manage groups screen.
1444         // make sure there is only one element
1445         // this is used when we are running under smog and need to know
1446         // how the contact was added (CRM-1203)
1447         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1448         if ((count($this->_paramLookup['group']) == 1) &&
1449           (count($groups) == 1)
1450         ) {
1451           $groupId = $groups[0];
1452
1453           //check if group is saved search
1454           $group = new CRM_Contact_BAO_Group();
1455           $group->id = $groupId;
1456           $group->find(TRUE);
1457
1458           if (!isset($group->saved_search_id)) {
1459             $tbName = "civicrm_group_contact";
1460             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1461             // as this will add load to an intentionally light query.
1462             // ideally this code would be removed as it appears to be to support CRM-1203
1463             // and passing in the required returnProperties from the url would
1464             // make more sense that globally applying the requirements of one form.
1465             if (($this->_returnProperties != ['contact_id'])) {
1466               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1467               $this->_element['group_contact_id'] = 1;
1468               $this->_select['status'] = "$tbName.status as status";
1469               $this->_element['status'] = 1;
1470             }
1471           }
1472         }
1473         $this->_useGroupBy = TRUE;
1474       }
1475       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1476         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1477           // CRM-5954
1478           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1479           $this->_useDistinct = FALSE;
1480           $this->_useGroupBy = TRUE;
1481         }
1482       }
1483
1484       $select = $this->getSelect();
1485       $from = $this->_fromClause;
1486     }
1487
1488     $where = '';
1489     if (!empty($this->_whereClause)) {
1490       $where = "WHERE {$this->_whereClause}";
1491     }
1492
1493     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1494       if (!empty($this->_permissionFromClause)) {
1495         $from .= " $this->_permissionFromClause";
1496       }
1497       if (empty($where)) {
1498         $where = "WHERE $this->_permissionWhereClause";
1499       }
1500       else {
1501         $where = "$where AND $this->_permissionWhereClause";
1502       }
1503     }
1504
1505     $having = '';
1506     if (!empty($this->_having)) {
1507       foreach ($this->_having as $havingSets) {
1508         foreach ($havingSets as $havingSet) {
1509           $havingValue[] = $havingSet;
1510         }
1511       }
1512       $having = ' HAVING ' . implode(' AND ', $havingValue);
1513     }
1514
1515     // if we are doing a transform, do it here
1516     // use the $from, $where and $having to get the contact ID
1517     if ($this->_displayRelationshipType) {
1518       $this->filterRelatedContacts($from, $where, $having);
1519     }
1520
1521     return [$select, $from, $where, $having];
1522   }
1523
1524   /**
1525    * Get where values from the parameters.
1526    *
1527    * @param string $name
1528    * @param mixed $grouping
1529    *
1530    * @return mixed
1531    */
1532   public function getWhereValues($name, $grouping) {
1533     $result = NULL;
1534     foreach ($this->_params as $values) {
1535       if ($values[0] == $name && $values[3] == $grouping) {
1536         return $values;
1537       }
1538     }
1539
1540     return $result;
1541   }
1542
1543   /**
1544    * Fix date values.
1545    *
1546    * @param bool $relative
1547    * @param string $from
1548    * @param string $to
1549    */
1550   public static function fixDateValues($relative, &$from, &$to) {
1551     if ($relative) {
1552       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1553     }
1554   }
1555
1556   /**
1557    * Convert values from form-appropriate to query-object appropriate.
1558    *
1559    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1560    * So, ideally we would convert all fields to look like
1561    *  array(
1562    *   0 => $fieldName
1563    *   // Set the operator for legacy reasons, but it is ignored
1564    *   1 =>  '='
1565    *   // array in sql filter syntax
1566    *   2 => array('BETWEEN' => array(1,60),
1567    *   3 => null
1568    *   4 => null
1569    *  );
1570    *
1571    * There are some examples of the syntax in
1572    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1573    *
1574    * More notes at CRM_Core_DAO::createSQLFilter
1575    *
1576    * and a list of supported operators in CRM_Core_DAO
1577    *
1578    * @param array $formValues
1579    * @param int $wildcard
1580    * @param bool $useEquals
1581    *
1582    * @param string $apiEntity
1583    *
1584    * @param array $entityReferenceFields
1585    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1586    *
1587    * @return array
1588    */
1589   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1590     $entityReferenceFields = []) {
1591     $params = [];
1592     if (empty($formValues)) {
1593       return $params;
1594     }
1595
1596     self::filterCountryFromValuesIfStateExists($formValues);
1597     CRM_Core_BAO_CustomValue::fixCustomFieldValue($formValues);
1598
1599     foreach ($formValues as $id => $values) {
1600       if (self::isAlreadyProcessedForQueryFormat($values)) {
1601         $params[] = $values;
1602         continue;
1603       }
1604
1605       self::legacyConvertFormValues($id, $values);
1606
1607       // The form uses 1 field to represent two db fields
1608       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1609         $contactType = [];
1610         $subType = [];
1611         foreach ((array) $values as $key => $type) {
1612           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1613           $contactType[$types[0]] = $types[0];
1614           // Add sub-type if specified
1615           if (!empty($types[1])) {
1616             $subType[$types[1]] = $types[1];
1617           }
1618         }
1619         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1620         if ($subType) {
1621           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1622         }
1623       }
1624       elseif ($id === 'privacy') {
1625         if (is_array($formValues['privacy'])) {
1626           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1627           foreach ($formValues['privacy'] as $key => $value) {
1628             if ($value) {
1629               $params[] = [$key, $op, $value, 0, 0];
1630             }
1631           }
1632         }
1633       }
1634       elseif ($id === 'email_on_hold') {
1635         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1636           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1637           // so in 5.11 we have an extra if that should become redundant over time.
1638           // https://lab.civicrm.org/dev/core/issues/745
1639           // @todo this renaming of email_on_hold to on_hold needs revisiting
1640           // it precedes recent changes but causes the default not to reload.
1641           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1642           if (!empty($onHoldValue)) {
1643             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1644           }
1645         }
1646       }
1647       elseif (substr($id, 0, 7) === 'custom_'
1648         &&  (
1649           substr($id, -5, 5) === '_from'
1650           || substr($id, -3, 3) === '_to'
1651         )
1652       ) {
1653         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1654       }
1655       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1656         FALSE)) {
1657         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1658       }
1659       else {
1660         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1661
1662         if (!$values) {
1663           continue;
1664         }
1665         $params[] = $values;
1666       }
1667     }
1668     return $params;
1669   }
1670
1671   /**
1672    * Function to support legacy format for groups and tags.
1673    *
1674    * @param string $id
1675    * @param array|int $values
1676    *
1677    */
1678   public static function legacyConvertFormValues($id, &$values) {
1679     $legacyElements = [
1680       'group',
1681       'tag',
1682       'contact_tags',
1683       'contact_type',
1684       'membership_type_id',
1685       'membership_status_id',
1686     ];
1687     if (in_array($id, $legacyElements) && is_array($values)) {
1688       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1689       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1690       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1691     }
1692   }
1693
1694   /**
1695    * Fix values from query from/to something no-one cared enough to document.
1696    *
1697    * @param int $id
1698    * @param array $values
1699    * @param int $wildcard
1700    * @param bool $useEquals
1701    *
1702    * @param string $apiEntity
1703    *
1704    * @return array|null
1705    */
1706   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1707     // skip a few search variables
1708     static $skipWhere = NULL;
1709     static $likeNames = NULL;
1710     $result = NULL;
1711
1712     // Change camelCase EntityName to lowercase with underscores
1713     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1714
1715     // check if $value is in OK (Operator as Key) format as used by Get API
1716     if (CRM_Utils_System::isNull($values)) {
1717       return $result;
1718     }
1719
1720     if (!$skipWhere) {
1721       $skipWhere = [
1722         'task',
1723         'radio_ts',
1724         'uf_group_id',
1725         'component_mode',
1726         'qfKey',
1727         'operator',
1728         'display_relationship_type',
1729       ];
1730     }
1731
1732     if (in_array($id, $skipWhere) ||
1733       substr($id, 0, 4) == '_qf_' ||
1734       substr($id, 0, 7) == 'hidden_'
1735     ) {
1736       return $result;
1737     }
1738
1739     if ($apiEntity &&
1740       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1741       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1742       (substr($id, 0, 7) != 'custom_')
1743     ) {
1744       $id = $apiEntity . '_' . $id;
1745     }
1746
1747     if (!$likeNames) {
1748       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1749     }
1750
1751     // email comes in via advanced search
1752     // so use wildcard always
1753     if ($id == 'email') {
1754       $wildcard = 1;
1755     }
1756
1757     if (!$useEquals && in_array($id, $likeNames)) {
1758       $result = [$id, 'LIKE', $values, 0, 1];
1759     }
1760     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1761       $result = [$id, 'LIKE', $values, 0, 0];
1762     }
1763     elseif ($id == 'contact_type' ||
1764       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1765     ) {
1766       $result = [$id, 'IN', $values, 0, $wildcard];
1767     }
1768     else {
1769       $result = [$id, '=', $values, 0, $wildcard];
1770     }
1771
1772     return $result;
1773   }
1774
1775   /**
1776    * Get the where clause for a single field.
1777    *
1778    * @param array $values
1779    * @param bool $isForcePrimaryOnly
1780    *
1781    * @throws \CRM_Core_Exception
1782    */
1783   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1784     if ($this->isARelativeDateField($values[0])) {
1785       $this->buildRelativeDateQuery($values);
1786       return;
1787     }
1788     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1789
1790     // do not process custom fields or prefixed contact ids or component params
1791     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1792       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1793       (substr($values[0], 0, 13) === 'contribution_') ||
1794       (substr($values[0], 0, 6) === 'event_') ||
1795       (substr($values[0], 0, 12) === 'participant_') ||
1796       (substr($values[0], 0, 7) === 'member_') ||
1797       (substr($values[0], 0, 6) === 'grant_') ||
1798       (substr($values[0], 0, 7) === 'pledge_') ||
1799       (substr($values[0], 0, 5) === 'case_') ||
1800       (substr($values[0], 0, 10) === 'financial_') ||
1801       (substr($values[0], 0, 8) === 'payment_') ||
1802       (substr($values[0], 0, 11) === 'membership_')
1803       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1804       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1805       // be needed.
1806       || $values[0] === 'mailing_id'
1807     ) {
1808       return;
1809     }
1810
1811     // skip for hook injected fields / params
1812     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1813     if (array_key_exists($values[0], $extFields)) {
1814       return;
1815     }
1816
1817     switch ($values[0]) {
1818       case 'deleted_contacts':
1819         $this->deletedContacts($values);
1820         return;
1821
1822       case 'contact_sub_type':
1823         $this->contactSubType($values);
1824         return;
1825
1826       case 'group':
1827       case 'group_type':
1828         $this->group($values);
1829         return;
1830
1831       // case tag comes from find contacts
1832       case 'tag_search':
1833         $this->tagSearch($values);
1834         return;
1835
1836       case 'tag':
1837       case 'contact_tags':
1838         $this->tag($values);
1839         return;
1840
1841       case 'note':
1842       case 'note_body':
1843       case 'note_subject':
1844         $this->notes($values);
1845         return;
1846
1847       case 'uf_user':
1848         $this->ufUser($values);
1849         return;
1850
1851       case 'sort_name':
1852       case 'display_name':
1853         $this->sortName($values);
1854         return;
1855
1856       case 'addressee':
1857       case 'postal_greeting':
1858       case 'email_greeting':
1859         $this->greetings($values);
1860         return;
1861
1862       case 'email':
1863       case 'email_id':
1864         $this->email($values, $isForcePrimaryOnly);
1865         return;
1866
1867       case 'phone_numeric':
1868         $this->phone_numeric($values);
1869         return;
1870
1871       case 'phone_phone_type_id':
1872       case 'phone_location_type_id':
1873         $this->phone_option_group($values);
1874         return;
1875
1876       case 'street_address':
1877         $this->street_address($values);
1878         return;
1879
1880       case 'street_number':
1881         $this->street_number($values);
1882         return;
1883
1884       case 'sortByCharacter':
1885         $this->sortByCharacter($values);
1886         return;
1887
1888       case 'location_type':
1889         $this->locationType($values);
1890         return;
1891
1892       case 'county':
1893         $this->county($values);
1894         return;
1895
1896       case 'state_province':
1897       case 'state_province_id':
1898       case 'state_province_name':
1899         $this->stateProvince($values);
1900         return;
1901
1902       case 'country':
1903       case 'country_id':
1904         $this->country($values, FALSE);
1905         return;
1906
1907       case 'postal_code':
1908       case 'postal_code_low':
1909       case 'postal_code_high':
1910         $this->postalCode($values);
1911         return;
1912
1913       case 'activity_date':
1914       case 'activity_date_low':
1915       case 'activity_date_high':
1916       case 'activity_date_time_low':
1917       case 'activity_date_time_high':
1918       case 'activity_role':
1919       case 'activity_status_id':
1920       case 'activity_status':
1921       case 'activity_priority':
1922       case 'activity_priority_id':
1923       case 'followup_parent_id':
1924       case 'parent_id':
1925       case 'source_contact_id':
1926       case 'activity_text':
1927       case 'activity_option':
1928       case 'test_activities':
1929       case 'activity_type_id':
1930       case 'activity_type':
1931       case 'activity_survey_id':
1932       case 'activity_tags':
1933       case 'activity_taglist':
1934       case 'activity_test':
1935       case 'activity_campaign_id':
1936       case 'activity_engagement_level':
1937       case 'activity_id':
1938       case 'activity_result':
1939       case 'source_contact':
1940         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1941         return;
1942
1943       case 'age_low':
1944       case 'age_high':
1945       case 'birth_date_low':
1946       case 'birth_date_high':
1947       case 'deceased_date_low':
1948       case 'deceased_date_high':
1949         $this->demographics($values);
1950         return;
1951
1952       case 'age_asof_date':
1953         // handled by demographics
1954         return;
1955
1956       case 'log_date_low':
1957       case 'log_date_high':
1958         $this->modifiedDates($values);
1959         return;
1960
1961       case 'changed_by':
1962         $this->changeLog($values);
1963         return;
1964
1965       case 'do_not_phone':
1966       case 'do_not_email':
1967       case 'do_not_mail':
1968       case 'do_not_sms':
1969       case 'do_not_trade':
1970       case 'is_opt_out':
1971         $this->privacy($values);
1972         return;
1973
1974       case 'privacy_options':
1975         $this->privacyOptions($values);
1976         return;
1977
1978       case 'privacy_operator':
1979       case 'privacy_toggle':
1980         // these are handled by privacy options
1981         return;
1982
1983       case 'preferred_communication_method':
1984         $this->preferredCommunication($values);
1985         return;
1986
1987       case 'relation_type_id':
1988       case 'relationship_start_date_high':
1989       case 'relationship_start_date_low':
1990       case 'relationship_end_date_high':
1991       case 'relationship_end_date_low':
1992       case 'relation_active_period_date_high':
1993       case 'relation_active_period_date_low':
1994       case 'relation_target_name':
1995       case 'relation_status':
1996       case 'relation_description':
1997       case 'relation_date_low':
1998       case 'relation_date_high':
1999         $this->relationship($values);
2000         $this->_relationshipValuesAdded = TRUE;
2001         return;
2002
2003       case 'task_status_id':
2004         $this->task($values);
2005         return;
2006
2007       case 'task_id':
2008         // since this case is handled with the above
2009         return;
2010
2011       case 'prox_distance':
2012         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2013         return;
2014
2015       case 'prox_street_address':
2016       case 'prox_city':
2017       case 'prox_postal_code':
2018       case 'prox_state_province_id':
2019       case 'prox_country_id':
2020       case 'prox_geo_code_1':
2021       case 'prox_geo_code_2':
2022         // handled by the proximity_distance clause
2023         return;
2024
2025       default:
2026         $this->restWhere($values);
2027         return;
2028     }
2029   }
2030
2031   /**
2032    * Given a list of conditions in params generate the required where clause.
2033    *
2034    * @param bool $isForcePrimaryEmailOnly
2035    *
2036    * @return string
2037    * @throws \CRM_Core_Exception
2038    */
2039   public function whereClause($isForcePrimaryEmailOnly = NULL) {
2040     $this->_where[0] = [];
2041     $this->_qill[0] = [];
2042
2043     $this->includeContactIDs();
2044     if (!empty($this->_params)) {
2045       foreach (array_keys($this->_params) as $id) {
2046         if (empty($this->_params[$id][0])) {
2047           continue;
2048         }
2049         // check for both id and contact_id
2050         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2051           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2052           $field = $this->_fields['id'] ?? NULL;
2053           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2054             'CRM_Contact_BAO_Contact',
2055             "contact_a.id",
2056             $this->_params[$id][2],
2057             $this->_params[$id][1]
2058           );
2059           $this->_qill[0][] = ts("%1 %2 %3", [
2060             1 => $field['title'] ?? '',
2061             2 => $qillop,
2062             3 => $qillVal,
2063           ]);
2064         }
2065         else {
2066           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2067         }
2068       }
2069
2070       CRM_Core_Component::alterQuery($this, 'where');
2071
2072       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2073     }
2074
2075     if ($this->_customQuery) {
2076       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2077       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2078       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2079       if (!empty($this->_customQuery->_where)) {
2080         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2081       }
2082       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2083     }
2084
2085     $clauses = [];
2086     $andClauses = [];
2087
2088     $validClauses = 0;
2089     if (!empty($this->_where)) {
2090       foreach ($this->_where as $grouping => $values) {
2091         if ($grouping > 0 && !empty($values)) {
2092           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2093           $validClauses++;
2094         }
2095       }
2096
2097       if (!empty($this->_where[0])) {
2098         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2099       }
2100       if (!empty($clauses)) {
2101         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2102       }
2103
2104       if ($validClauses > 1) {
2105         $this->_useDistinct = TRUE;
2106       }
2107     }
2108
2109     return implode(' AND ', $andClauses);
2110   }
2111
2112   /**
2113    * Generate where clause for any parameters not already handled.
2114    *
2115    * @param array $values
2116    *
2117    * @throws Exception
2118    */
2119   public function restWhere(&$values) {
2120     $name = $values[0] ?? NULL;
2121     $op = $values[1] ?? NULL;
2122     $value = $values[2] ?? NULL;
2123     $grouping = $values[3] ?? NULL;
2124     $wildcard = $values[4] ?? NULL;
2125
2126     if (isset($grouping) && empty($this->_where[$grouping])) {
2127       $this->_where[$grouping] = [];
2128     }
2129
2130     $multipleFields = ['url'];
2131
2132     //check if the location type exists for fields
2133     $lType = '';
2134     $locType = explode('-', $name);
2135
2136     if (!in_array($locType[0], $multipleFields)) {
2137       //add phone type if exists
2138       if (isset($locType[2]) && $locType[2]) {
2139         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2140       }
2141     }
2142
2143     $field = $this->_fields[$name] ?? NULL;
2144
2145     if (!$field) {
2146       $field = $this->_fields[$locType[0]] ?? NULL;
2147
2148       if (!$field) {
2149         // Strip any trailing _high & _low that might be appended.
2150         $realFieldName = str_replace(['_high', '_low'], '', $name);
2151         if (isset($this->_fields[$realFieldName])) {
2152           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2153           $columnName = $field['column_name'] ?? $field['name'];
2154           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2155         }
2156         return;
2157       }
2158     }
2159
2160     $setTables = TRUE;
2161
2162     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2163     if (isset($locType[1]) && is_numeric($locType[1])) {
2164       $lType = $locationType[$locType[1]];
2165     }
2166     if ($lType) {
2167       $field['title'] .= " ($lType)";
2168     }
2169
2170     if (substr($name, 0, 14) === 'state_province') {
2171       if (isset($locType[1]) && is_numeric($locType[1])) {
2172         $setTables = FALSE;
2173         $aName = "{$lType}-address";
2174         $where = "`$aName`.state_province_id";
2175       }
2176       else {
2177         $where = "civicrm_address.state_province_id";
2178       }
2179
2180       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2181       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2182       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2183       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2184     }
2185     elseif (!empty($field['pseudoconstant'])) {
2186       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2187       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2188       $this->optionValueQuery(
2189         $name, $op, $value, $grouping,
2190         'CRM_Contact_DAO_Contact',
2191         $field,
2192         $field['html']['label'] ?? $field['title'],
2193         CRM_Utils_Type::typeToString($dataType)
2194       );
2195       if ($name === 'gender_id') {
2196         self::$_openedPanes[ts('Demographics')] = TRUE;
2197       }
2198     }
2199     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2200       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2201       if (isset($locType[1]) && is_numeric($locType[1])) {
2202         $setTables = FALSE;
2203         $aName = "{$lType}-address";
2204         $where = "`$aName`.$name";
2205       }
2206       else {
2207         $where = "civicrm_address.$name";
2208       }
2209
2210       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2211       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2212
2213       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2214       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2215     }
2216     elseif ($name === 'world_region') {
2217       $this->optionValueQuery(
2218         $name, $op, $value, $grouping,
2219         NULL,
2220         $field,
2221         ts('World Region'),
2222         'Positive'
2223       );
2224     }
2225     elseif ($name === 'is_deceased') {
2226       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2227       self::$_openedPanes[ts('Demographics')] = TRUE;
2228     }
2229     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2230       $appendDateTime = TRUE;
2231       if ($name === 'deceased_date' || $name === 'birth_date') {
2232         $appendDateTime = FALSE;
2233         self::$_openedPanes[ts('Demographics')] = TRUE;
2234       }
2235       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2236     }
2237     elseif ($name === 'contact_id') {
2238       if (is_int($value)) {
2239         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2240         $this->_qill[$grouping][] = "$field[title] $op $value";
2241       }
2242     }
2243     elseif ($name === 'name') {
2244       $value = CRM_Core_DAO::escapeString($value);
2245       if ($wildcard) {
2246         $op = 'LIKE';
2247         $value = self::getWildCardedValue($wildcard, $op, $value);
2248       }
2249       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2250       // @todo it's likely this code path is obsolete / never called. It is definitely not
2251       // passed through in our test suite.
2252       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2253       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2254     }
2255     elseif ($name === 'current_employer') {
2256       if ($wildcard) {
2257         $op = 'LIKE';
2258         $value = self::getWildCardedValue($wildcard, $op, $value);
2259       }
2260       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2261         $value
2262       );
2263       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2264       $this->_where[$grouping][] = $ceWhereClause;
2265       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2266     }
2267     elseif (substr($name, 0, 4) === 'url-') {
2268       $tName = 'civicrm_website';
2269       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2270       $value = CRM_Core_DAO::escapeString($value);
2271       if ($wildcard) {
2272         $op = 'LIKE';
2273         $value = self::getWildCardedValue($wildcard, $op, $value);
2274       }
2275
2276       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2277       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2278     }
2279     elseif ($name === 'contact_is_deleted') {
2280       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2281     }
2282     elseif (!empty($field['where'])) {
2283       $type = NULL;
2284       if (!empty($field['type'])) {
2285         $type = CRM_Utils_Type::typeToString($field['type']);
2286       }
2287
2288       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2289
2290       if (isset($locType[1]) &&
2291         is_numeric($locType[1])
2292       ) {
2293         $setTables = FALSE;
2294
2295         //get the location name
2296         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2297         $fieldName = "`$tName`.$fldName";
2298
2299         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2300         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2301
2302       }
2303       else {
2304         if ($tableName == 'civicrm_contact') {
2305           $fieldName = "contact_a.{$fieldName}";
2306         }
2307         else {
2308           $fieldName = $field['where'];
2309         }
2310       }
2311
2312       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2313       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2314         1 => $field['title'],
2315         2 => $qillop,
2316         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2317       ]);
2318
2319       if (is_array($value)) {
2320         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2321         // for api style operators for functions that hit this point without worrying about regression
2322         // (the previous comments indicated the condition for hitting this point were unknown
2323         // per CRM-14743 we are adding modified_date & created_date operator support
2324         $operations = array_keys($value);
2325         foreach ($operations as $operator) {
2326           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2327             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2328             //so this condition will satisfy the search for now
2329             if (strpos($op, 'IN') !== FALSE) {
2330               $value = [$op => $value];
2331             }
2332             // we don't know when this might happen
2333             else {
2334               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2335             }
2336           }
2337         }
2338         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2339       }
2340       else {
2341         if ($wildcard) {
2342           $op = 'LIKE';
2343           $value = self::getWildCardedValue($wildcard, $op, $value);
2344         }
2345
2346         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2347       }
2348     }
2349
2350     if ($setTables && isset($field['where'])) {
2351       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2352       if (isset($tableName)) {
2353         $this->_tables[$tableName] = 1;
2354         $this->_whereTables[$tableName] = 1;
2355       }
2356     }
2357   }
2358
2359   /**
2360    * @param $where
2361    * @param $locType
2362    *
2363    * @return array
2364    * @throws Exception
2365    */
2366   public static function getLocationTableName(&$where, &$locType) {
2367     if (isset($locType[1]) && is_numeric($locType[1])) {
2368       list($tbName, $fldName) = explode(".", $where);
2369
2370       //get the location name
2371       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2372       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2373       if (in_array($locType[0], $specialFields)) {
2374         //hack to fix / special handing for phone_ext
2375         if ($locType[0] == 'phone_ext') {
2376           $locType[0] = 'phone';
2377         }
2378         if (isset($locType[2]) && $locType[2]) {
2379           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2380         }
2381         else {
2382           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2383         }
2384       }
2385       elseif (in_array($locType[0],
2386         [
2387           'address_name',
2388           'street_address',
2389           'street_name',
2390           'street_number_suffix',
2391           'street_unit',
2392           'supplemental_address_1',
2393           'supplemental_address_2',
2394           'supplemental_address_3',
2395           'city',
2396           'postal_code',
2397           'postal_code_suffix',
2398           'geo_code_1',
2399           'geo_code_2',
2400           'master_id',
2401         ]
2402       )) {
2403         //fix for search by profile with address fields.
2404         $tName = "{$locationType[$locType[1]]}-address";
2405       }
2406       elseif (in_array($locType[0],
2407           [
2408             'on_hold',
2409             'signature_html',
2410             'signature_text',
2411             'is_bulkmail',
2412           ]
2413         )) {
2414         $tName = "{$locationType[$locType[1]]}-email";
2415       }
2416       elseif ($locType[0] == 'provider_id') {
2417         $tName = "{$locationType[$locType[1]]}-im";
2418       }
2419       elseif ($locType[0] == 'openid') {
2420         $tName = "{$locationType[$locType[1]]}-openid";
2421       }
2422       else {
2423         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2424       }
2425       $tName = str_replace(' ', '_', $tName);
2426       return [$tName, $fldName];
2427     }
2428     throw new CRM_Core_Exception('Cannot determine location table information');
2429   }
2430
2431   /**
2432    * Given a result dao, extract the values and return that array
2433    *
2434    * @param CRM_Core_DAO $dao
2435    *
2436    * @return array
2437    *   values for this query
2438    */
2439   public function store($dao) {
2440     $value = [];
2441
2442     foreach ($this->_element as $key => $dontCare) {
2443       if (property_exists($dao, $key)) {
2444         if (strpos($key, '-') !== FALSE) {
2445           $values = explode('-', $key);
2446           $lastElement = array_pop($values);
2447           $current = &$value;
2448           $cnt = count($values);
2449           $count = 1;
2450           foreach ($values as $v) {
2451             if (!array_key_exists($v, $current)) {
2452               $current[$v] = [];
2453             }
2454             //bad hack for im_provider
2455             if ($lastElement == 'provider_id') {
2456               if ($count < $cnt) {
2457                 $current = &$current[$v];
2458               }
2459               else {
2460                 $lastElement = "{$v}_{$lastElement}";
2461               }
2462             }
2463             else {
2464               $current = &$current[$v];
2465             }
2466             $count++;
2467           }
2468
2469           $current[$lastElement] = $dao->$key;
2470         }
2471         else {
2472           $value[$key] = $dao->$key;
2473         }
2474       }
2475     }
2476     return $value;
2477   }
2478
2479   /**
2480    * Getter for tables array.
2481    *
2482    * @return array
2483    */
2484   public function tables() {
2485     return $this->_tables;
2486   }
2487
2488   /**
2489    * Sometimes used to create the from clause, but, not reliably, set
2490    * this AND set tables.
2491    *
2492    * It's unclear the intent - there is a 'simpleFrom' clause which
2493    * takes whereTables into account & a fromClause which doesn't.
2494    *
2495    * logic may have eroded?
2496    *
2497    * @return array
2498    */
2499   public function whereTables() {
2500     return $this->_whereTables;
2501   }
2502
2503   /**
2504    * Generate the where clause (used in match contacts and permissions)
2505    *
2506    * @param array $params
2507    * @param array $fields
2508    * @param array $tables
2509    * @param $whereTables
2510    * @param bool $strict
2511    *
2512    * @return string
2513    * @throws \CRM_Core_Exception
2514    */
2515   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2516     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2517       FALSE, $strict
2518     );
2519
2520     $tables = array_merge($query->tables(), $tables);
2521     $whereTables = array_merge($query->whereTables(), $whereTables);
2522
2523     return $query->_whereClause;
2524   }
2525
2526   /**
2527    * Create the from clause.
2528    *
2529    * @param array $tables
2530    *   Tables that need to be included in this from clause. If null,
2531    *   return mimimal from clause (i.e. civicrm_contact).
2532    * @param array $inner
2533    *   Tables that should be inner-joined.
2534    * @param array $right
2535    *   Tables that should be right-joined.
2536    * @param bool $primaryLocation
2537    *   Search on primary location. See note below.
2538    * @param int $mode
2539    *   Determines search mode based on bitwise MODE_* constants.
2540    * @param string|NULL $apiEntity
2541    *   Determines search mode based on entity by string.
2542    *
2543    * The $primaryLocation flag only seems to be used when
2544    * locationType() has been called. This may be a search option
2545    * exposed, or perhaps it's a "search all details" approach which
2546    * predates decoupling of location types and primary fields?
2547    *
2548    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2549    *
2550    * @return string
2551    *   the from clause
2552    */
2553   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2554
2555     $from = ' FROM civicrm_contact contact_a';
2556     if (empty($tables)) {
2557       return $from;
2558     }
2559
2560     if (!empty($tables['civicrm_worldregion'])) {
2561       $tables = array_merge(['civicrm_country' => 1], $tables);
2562     }
2563
2564     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2565         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2566       $tables = array_merge(['civicrm_address' => 1],
2567         $tables
2568       );
2569     }
2570
2571     // add group_contact and group table is subscription history is present
2572     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2573       $tables = array_merge([
2574         'civicrm_group' => 1,
2575         'civicrm_group_contact' => 1,
2576       ],
2577         $tables
2578       );
2579     }
2580
2581     // to handle table dependencies of components
2582     CRM_Core_Component::tableNames($tables);
2583     // to handle table dependencies of hook injected tables
2584     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2585
2586     //format the table list according to the weight
2587     $info = CRM_Core_TableHierarchy::info();
2588
2589     foreach ($tables as $key => $value) {
2590       $k = 99;
2591       if (strpos($key, '-') !== FALSE) {
2592         $keyArray = explode('-', $key);
2593         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2594       }
2595       elseif (strpos($key, '_') !== FALSE) {
2596         $keyArray = explode('_', $key);
2597         if (is_numeric(array_pop($keyArray))) {
2598           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2599         }
2600         else {
2601           $k = CRM_Utils_Array::value($key, $info, 99);
2602         }
2603       }
2604       else {
2605         $k = CRM_Utils_Array::value($key, $info, 99);
2606       }
2607       $tempTable[$k . ".$key"] = $key;
2608     }
2609     ksort($tempTable);
2610     $newTables = [];
2611     foreach ($tempTable as $key) {
2612       $newTables[$key] = $tables[$key];
2613     }
2614
2615     $tables = $newTables;
2616
2617     foreach ($tables as $name => $value) {
2618       if (!$value) {
2619         continue;
2620       }
2621
2622       if (!empty($inner[$name])) {
2623         $side = 'INNER';
2624       }
2625       elseif (!empty($right[$name])) {
2626         $side = 'RIGHT';
2627       }
2628       else {
2629         $side = 'LEFT';
2630       }
2631
2632       if ($value != 1) {
2633         // if there is already a join statement in value, use value itself
2634         if (strpos($value, 'JOIN')) {
2635           $from .= " $value ";
2636         }
2637         else {
2638           $from .= " $side JOIN $name ON ( $value ) ";
2639         }
2640         continue;
2641       }
2642
2643       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2644     }
2645     return $from;
2646   }
2647
2648   /**
2649    * Get join statements for the from clause depending on entity type
2650    *
2651    * @param string $name
2652    * @param int $mode
2653    * @param string $side
2654    * @param string $primaryLocation
2655    * @return string
2656    */
2657   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2658     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2659     switch ($name) {
2660       case 'civicrm_address':
2661         //CRM-14263 further handling of address joins further down...
2662         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2663
2664       case 'civicrm_state_province':
2665         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2666         // but that will have been implied-in by the calling function.
2667         // test cover in testContactIDQuery
2668         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2669
2670       case 'civicrm_country':
2671         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2672         // but that will have been implied-in by the calling function.
2673         // test cover in testContactIDQuery
2674         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2675
2676       case 'civicrm_phone':
2677         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2678
2679       case 'civicrm_email':
2680         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2681
2682       case 'civicrm_im':
2683         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2684
2685       case 'im_provider':
2686         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2687         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2688         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2689         return $from;
2690
2691       case 'civicrm_openid':
2692         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2693
2694       case 'civicrm_worldregion':
2695         // We can be sure from the calling function that country will already be joined in.
2696         // we really don't need world_region - we could use a pseudoconstant for it.
2697         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2698
2699       case 'civicrm_location_type':
2700         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2701
2702       case 'civicrm_group':
2703         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2704
2705       case 'civicrm_group_contact':
2706         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2707
2708       case 'civicrm_group_contact_cache':
2709         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2710
2711       case 'civicrm_activity':
2712       case 'civicrm_activity_tag':
2713       case 'activity_type':
2714       case 'activity_status':
2715       case 'parent_id':
2716       case 'civicrm_activity_contact':
2717       case 'source_contact':
2718       case 'activity_priority':
2719         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2720
2721       case 'civicrm_entity_tag':
2722         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2723         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2724
2725       case 'civicrm_note':
2726         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2727         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2728
2729       case 'civicrm_subscription_history':
2730         $from = " $side JOIN civicrm_subscription_history";
2731         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2732         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2733
2734       case 'civicrm_relationship':
2735         if (self::$_relType == 'reciprocal') {
2736           if (self::$_relationshipTempTable) {
2737             // we have a temptable to join on
2738             $tbl = self::$_relationshipTempTable;
2739             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2740           }
2741           else {
2742             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2743             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2744             return $from;
2745           }
2746         }
2747         elseif (self::$_relType == 'b') {
2748           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2749           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2750         }
2751         else {
2752           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2753           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2754         }
2755
2756       case 'civicrm_log':
2757         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2758         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2759
2760       case 'civicrm_tag':
2761         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2762
2763       case 'civicrm_grant':
2764         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2765
2766       case 'civicrm_website':
2767         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2768
2769       case 'civicrm_campaign':
2770         //Move to default case if not in either mode.
2771         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2772           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2773         }
2774         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2775           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2776         }
2777         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2778           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2779         }
2780
2781       default:
2782         $locationTypeName = '';
2783         if (strpos($name, '-address') != 0) {
2784           $locationTypeName = 'address';
2785         }
2786         elseif (strpos($name, '-phone') != 0) {
2787           $locationTypeName = 'phone';
2788         }
2789         elseif (strpos($name, '-email') != 0) {
2790           $locationTypeName = 'email';
2791         }
2792         elseif (strpos($name, '-im') != 0) {
2793           $locationTypeName = 'im';
2794         }
2795         elseif (strpos($name, '-openid') != 0) {
2796           $locationTypeName = 'openid';
2797         }
2798
2799         if ($locationTypeName) {
2800           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2801           $parts = explode('-', $name);
2802           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2803           foreach ($locationTypes as $locationTypeID => $locationType) {
2804             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2805               $locationID = $locationTypeID;
2806             }
2807           }
2808           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2809         }
2810         else {
2811           $from = CRM_Core_Component::from($name, $mode, $side);
2812         }
2813         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2814
2815         return $from;
2816     }
2817   }
2818
2819   /**
2820    * WHERE / QILL clause for deleted_contacts
2821    *
2822    * @param array $values
2823    */
2824   public function deletedContacts($values) {
2825     list($_, $_, $value, $grouping, $_) = $values;
2826     if ($value) {
2827       // *prepend* to the relevant grouping as this is quite an important factor
2828       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2829     }
2830   }
2831
2832   /**
2833    * Where / qill clause for contact_type
2834    *
2835    * @param $values
2836    *
2837    * @throws \CRM_Core_Exception
2838    */
2839   public function contactType(&$values) {
2840     list($name, $op, $value, $grouping, $wildcard) = $values;
2841
2842     $subTypes = [];
2843     $clause = [];
2844
2845     // account for search builder mapping multiple values
2846     if (!is_array($value)) {
2847       $values = self::parseSearchBuilderString($value, 'String');
2848       if (is_array($values)) {
2849         $value = array_flip($values);
2850       }
2851     }
2852
2853     if (is_array($value)) {
2854       foreach ($value as $k => $v) {
2855         // fix for CRM-771
2856         if ($k) {
2857           $subType = NULL;
2858           $contactType = $k;
2859           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2860             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2861           }
2862
2863           if (!empty($subType)) {
2864             $subTypes[$subType] = 1;
2865           }
2866           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2867         }
2868       }
2869     }
2870     else {
2871       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2872       $contactType = $contactTypeANDSubType[0];
2873       $subType = $contactTypeANDSubType[1] ?? NULL;
2874       if (!empty($subType)) {
2875         $subTypes[$subType] = 1;
2876       }
2877       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2878     }
2879
2880     // fix for CRM-771
2881     if (!empty($clause)) {
2882       $quill = $clause;
2883       if ($op == 'IN' || $op == 'NOT IN') {
2884         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2885       }
2886       else {
2887         $type = array_pop($clause);
2888         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2889       }
2890
2891       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2892
2893       if (!empty($subTypes)) {
2894         $this->includeContactSubTypes($subTypes, $grouping);
2895       }
2896     }
2897   }
2898
2899   /**
2900    * Where / qill clause for contact_sub_type.
2901    *
2902    * @param array $values
2903    */
2904   public function contactSubType(&$values) {
2905     list($name, $op, $value, $grouping, $wildcard) = $values;
2906     $this->includeContactSubTypes($value, $grouping, $op);
2907   }
2908
2909   /**
2910    * @param $value
2911    * @param $grouping
2912    * @param string $op
2913    *
2914    * @throws \CRM_Core_Exception
2915    */
2916   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2917
2918     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2919       $op = key($value);
2920       $value = $value[$op];
2921     }
2922
2923     $clause = [];
2924     $alias = "contact_a.contact_sub_type";
2925     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2926
2927     $op = str_replace('IN', 'LIKE', $op);
2928     $op = str_replace('=', 'LIKE', $op);
2929     $op = str_replace('!', 'NOT ', $op);
2930
2931     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2932       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2933     }
2934     elseif (is_array($value)) {
2935       foreach ($value as $k => $v) {
2936         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2937       }
2938     }
2939     else {
2940       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2941     }
2942
2943     if (!empty($clause)) {
2944       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2945     }
2946     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2947   }
2948
2949   /**
2950    * Where / qill clause for groups.
2951    *
2952    * @param $values
2953    *
2954    * @throws \CRM_Core_Exception
2955    * @throws \Exception
2956    */
2957   public function group($values) {
2958     list($name, $op, $value, $grouping, $wildcard) = $values;
2959
2960     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2961     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2962       $op = key($value);
2963       $value = $value[$op];
2964     }
2965     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2966     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2967     // even one group might equate to multiple when looking at children so IN is simpler.
2968     // @todo - also look at != casting but there are rows below to review.
2969     $opReplacements = [
2970       'IS EMPTY' => 'IS NULL',
2971       'IS NOT EMPTY' => 'IS NOT NULL',
2972       '=' => 'IN',
2973     ];
2974     if (isset($opReplacements[$op])) {
2975       $op = $opReplacements[$op];
2976     }
2977
2978     if (strpos($op, 'NULL')) {
2979       $value = NULL;
2980     }
2981
2982     if (is_array($value) && count($value) > 1) {
2983       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2984         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2985       }
2986       $this->_useDistinct = TRUE;
2987     }
2988
2989     if (isset($value)) {
2990       $value = CRM_Utils_Array::value($op, $value, $value);
2991     }
2992
2993     if ($name === 'group_type') {
2994       $value = array_keys($this->getGroupsFromTypeCriteria($value));
2995     }
2996
2997     $regularGroupIDs = $smartGroupIDs = [];
2998     foreach ((array) $value as $id) {
2999       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3000         $smartGroupIDs[] = (int) $id;
3001       }
3002       else {
3003         $regularGroupIDs[] = (int) trim($id);
3004       }
3005     }
3006     $hasNonSmartGroups = count($regularGroupIDs);
3007
3008     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3009
3010     $statusJoinClause = $this->getGroupStatusClause($grouping);
3011     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3012     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3013     $groupClause = [];
3014     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3015       // include child groups IDs if any
3016       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3017       foreach ($childGroupIds as $key => $id) {
3018         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3019           $smartGroupIDs[] = $id;
3020           unset($childGroupIds[$key]);
3021         }
3022       }
3023       if (count($childGroupIds)) {
3024         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3025       }
3026
3027       if (empty($regularGroupIDs)) {
3028         if ($isGroupStatusSearch) {
3029           $regularGroupIDs = $smartGroupIDs;
3030         }
3031         // If it is still empty we want a filter that blocks all results.
3032         if (empty($regularGroupIDs)) {
3033           $regularGroupIDs = [0];
3034         }
3035       }
3036
3037       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3038       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3039
3040       // @todo consider just casting != to NOT IN & handling both together.
3041       if ($op === '!=') {
3042         $groupIds = '';
3043         if (!empty($regularGroupIDs)) {
3044           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3045         }
3046         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3047       }
3048       else {
3049         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3050       }
3051       $groupClause[] = "( {$clause} )";
3052
3053       if ($statusJoinClause) {
3054         $joinClause[] = "{$gcTable}.$statusJoinClause";
3055       }
3056       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3057     }
3058
3059     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3060     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3061     // as that info is in the group_contact table.
3062     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3063       $this->_groupUniqueKey = uniqid();
3064       $this->_groupKeys[] = $this->_groupUniqueKey;
3065       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3066       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3067       if (!empty($groupContactCacheClause)) {
3068         if ($isNotOp) {
3069           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3070           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3071           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3072           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3073           if (strpos($op, 'IN') !== FALSE) {
3074             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3075           }
3076           else {
3077             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3078           }
3079         }
3080         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3081       }
3082     }
3083
3084     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3085     if (!empty($groupClause)) {
3086       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3087     }
3088
3089     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3090     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3091     if (strpos($op, 'NULL') === FALSE) {
3092       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3093     }
3094   }
3095
3096   /**
3097    * @return array
3098    */
3099   public function getGroupCacheTableKeys() {
3100     return $this->_groupKeys;
3101   }
3102
3103   /**
3104    * Function translates selection of group type into a list of groups.
3105    * @param $value
3106    *
3107    * @return array
3108    */
3109   public function getGroupsFromTypeCriteria($value) {
3110     $groupIds = [];
3111     foreach ((array) $value as $groupTypeValue) {
3112       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3113       $groupIds = ($groupIds + $groupList);
3114     }
3115     return $groupIds;
3116   }
3117
3118   /**
3119    * Prime smart group cache for smart groups in the search, and join
3120    * civicrm_group_contact_cache table into the query.
3121    *
3122    * @param array $groups IDs of groups specified in search criteria.
3123    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3124    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3125    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3126    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3127    *
3128    * @return string WHERE clause component for smart group criteria.
3129    * @throws \CRM_Core_Exception
3130    */
3131   public function addGroupContactCache($groups, $tableAlias, $joinTable = "contact_a", $op, $joinColumn = 'id') {
3132     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3133     $groupsIds = $groups;
3134
3135     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3136     if (!empty($operator[$op]) && is_array($groups)) {
3137       $op = $operator[$op];
3138     }
3139     if (!$isNullOp && !$groups) {
3140       return NULL;
3141     }
3142     elseif (strpos($op, 'IN') !== FALSE) {
3143       $groups = [$op => $groups];
3144     }
3145     elseif (is_array($groups) && count($groups)) {
3146       $groups = ['IN' => $groups];
3147     }
3148
3149     // Find all the groups that are part of a saved search.
3150     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3151     $sql = "
3152 SELECT id, cache_date, saved_search_id, children
3153 FROM   civicrm_group
3154 WHERE  $smartGroupClause
3155   AND  ( saved_search_id != 0
3156    OR    saved_search_id IS NOT NULL
3157    OR    children IS NOT NULL )
3158 ";
3159
3160     $group = CRM_Core_DAO::executeQuery($sql);
3161
3162     while ($group->fetch()) {
3163       $this->_useDistinct = TRUE;
3164       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3165         CRM_Contact_BAO_GroupContactCache::load($group);
3166       }
3167     }
3168     if ($group->N == 0 && $op != 'NOT IN') {
3169       return NULL;
3170     }
3171
3172     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3173
3174     if ($op == 'NOT IN') {
3175       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3176     }
3177     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3178   }
3179
3180   /**
3181    * Where / qill clause for cms users
3182    *
3183    * @param $values
3184    */
3185   public function ufUser(&$values) {
3186     list($name, $op, $value, $grouping, $wildcard) = $values;
3187
3188     if ($value == 1) {
3189       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3190
3191       $this->_qill[$grouping][] = ts('CMS User');
3192     }
3193     elseif ($value == 0) {
3194       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3195
3196       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3197       $this->_qill[$grouping][] = ts('Not a CMS User');
3198     }
3199   }
3200
3201   /**
3202    * All tag search specific.
3203    *
3204    * @param array $values
3205    *
3206    * @throws \CRM_Core_Exception
3207    */
3208   public function tagSearch(&$values) {
3209     list($name, $op, $value, $grouping, $wildcard) = $values;
3210
3211     $op = "LIKE";
3212     $value = "%{$value}%";
3213     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3214
3215     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3216     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3217
3218     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3219     $tTable = "`civicrm_tag-" . uniqid() . "`";
3220
3221     if ($useAllTagTypes[2]) {
3222       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3223         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3224             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3225
3226       // search tag in cases
3227       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3228       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3229       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3230         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3231             LEFT JOIN civicrm_case
3232             ON (civicrm_case_contact.case_id = civicrm_case.id
3233                 AND civicrm_case.is_deleted = 0 )
3234             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3235             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3236       // search tag in activities
3237       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3238       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3239       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3240       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3241
3242       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3243         = " LEFT JOIN civicrm_activity_contact
3244             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3245             LEFT JOIN civicrm_activity
3246             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3247             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3248             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3249             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3250
3251       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3252       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3253     }
3254     else {
3255       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3256       $tTable = "`civicrm_tag-" . uniqid() . "`";
3257       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3258       {$etTable}.entity_table = 'civicrm_contact' )
3259                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3260
3261       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3262       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3263     }
3264   }
3265
3266   /**
3267    * Where / qill clause for tag.
3268    *
3269    * @param array $values
3270    *
3271    * @throws \CRM_Core_Exception
3272    */
3273   public function tag(&$values) {
3274     list($name, $op, $value, $grouping, $wildcard) = $values;
3275
3276     // API/Search Builder format array(operator => array(values))
3277     if (is_array($value)) {
3278       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3279         $op = key($value);
3280         $value = $value[$op];
3281       }
3282       if (count($value) > 1) {
3283         $this->_useDistinct = TRUE;
3284       }
3285     }
3286
3287     if (strpos($op, 'NULL') || strpos($op, 'EMPTY')) {
3288       $value = NULL;
3289     }
3290
3291     $tagTree = CRM_Core_BAO_Tag::getChildTags();
3292     foreach ((array) $value as $tagID) {
3293       if (!empty($tagTree[$tagID])) {
3294         $value = array_unique(array_merge($value, $tagTree[$tagID]));
3295       }
3296     }
3297
3298     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3299
3300     // implode array, then remove all spaces
3301     $value = str_replace(' ', '', implode(',', (array) $value));
3302     if (!empty($value)) {
3303       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3304     }
3305
3306     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3307     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3308
3309     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3310
3311     if (!empty($useAllTagTypes[2])) {
3312       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3313         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3314
3315       // search tag in cases
3316       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3317       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3318       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3319
3320       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3321         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3322             LEFT JOIN civicrm_case
3323             ON (civicrm_case_contact.case_id = civicrm_case.id
3324                 AND civicrm_case.is_deleted = 0 )
3325             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3326       // search tag in activities
3327       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3328       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3329         = " LEFT JOIN civicrm_activity_contact
3330             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3331             LEFT JOIN civicrm_activity
3332             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3333             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3334             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3335
3336       // CRM-10338
3337       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3338         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3339       }
3340       else {
3341         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3342       }
3343     }
3344     else {
3345       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3346         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3347
3348       // CRM-10338
3349       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3350         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3351         $op = str_replace('EMPTY', 'NULL', $op);
3352         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3353       }
3354       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3355       elseif ($op == '!=') {
3356         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3357       }
3358       elseif ($op == '=' || strstr($op, 'IN')) {
3359         $op = ($op == '=') ? 'IN' : $op;
3360         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3361       }
3362     }
3363     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3364   }
3365
3366   /**
3367    * Where/qill clause for notes
3368    *
3369    * @param array $values
3370    *
3371    * @throws \CRM_Core_Exception
3372    */
3373   public function notes(&$values) {
3374     list($name, $op, $value, $grouping, $wildcard) = $values;
3375
3376     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3377     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3378     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3379
3380     $this->_useDistinct = TRUE;
3381
3382     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3383       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3384
3385     $n = trim($value);
3386     $value = CRM_Core_DAO::escapeString($n);
3387     if ($wildcard) {
3388       if (strpos($value, '%') === FALSE) {
3389         $value = "%$value%";
3390       }
3391       $op = 'LIKE';
3392     }
3393     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3394       $value = NULL;
3395     }
3396
3397     $label = NULL;
3398     $clauses = [];
3399     if ($noteOption % 2 == 0) {
3400       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3401       $label = ts('Note: Body Only');
3402     }
3403     if ($noteOption % 3 == 0) {
3404       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3405       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3406     }
3407     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3408     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3409     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3410   }
3411
3412   /**
3413    * @param string $name
3414    * @param $op
3415    * @param $grouping
3416    *
3417    * @return bool
3418    */
3419   public function nameNullOrEmptyOp($name, $op, $grouping) {
3420     switch ($op) {
3421       case 'IS NULL':
3422       case 'IS NOT NULL':
3423         $this->_where[$grouping][] = "contact_a.$name $op";
3424         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3425         return TRUE;
3426
3427       case 'IS EMPTY':
3428         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3429         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3430         return TRUE;
3431
3432       case 'IS NOT EMPTY':
3433         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3434         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3435         return TRUE;
3436
3437       default:
3438         return FALSE;
3439     }
3440   }
3441
3442   /**
3443    * Where / qill clause for sort_name
3444    *
3445    * @param array $values
3446    */
3447   public function sortName(&$values) {
3448     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3449
3450     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3451     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3452       return;
3453     }
3454
3455     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3456
3457     if (!strlen($value)) {
3458       return;
3459     }
3460
3461     $config = CRM_Core_Config::singleton();
3462
3463     $sub = [];
3464
3465     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3466     $subGlue = ' OR ';
3467
3468     $firstChar = substr($value, 0, 1);
3469     $lastChar = substr($value, -1, 1);
3470     $quotes = ["'", '"'];
3471     // If string is quoted, strip quotes and otherwise don't alter it
3472     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3473       $value = trim($value, implode('', $quotes));
3474     }
3475     // Replace spaces with wildcards for a LIKE operation
3476     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3477     // Also need to check if there is space in between sort name.
3478     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') === TRUE) {
3479       $value = str_replace(' ', '%', $value);
3480     }
3481     $value = CRM_Core_DAO::escapeString(trim($value));
3482     if (strlen($value)) {
3483       $fieldsub = [];
3484       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3485       if ($fieldName == 'sort_name') {
3486         $wc = "contact_a.sort_name";
3487       }
3488       else {
3489         $wc = "contact_a.display_name";
3490       }
3491       $fieldsub[] = " ( $wc $op $value )";
3492       if ($config->includeNickNameInName) {
3493         $wc = "contact_a.nick_name";
3494         $fieldsub[] = " ( $wc $op $value )";
3495       }
3496       if ($config->includeEmailInName) {
3497         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3498       }
3499       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3500     }
3501
3502     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3503
3504     $this->_where[$grouping][] = $sub;
3505     if ($config->includeEmailInName) {
3506       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3507       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3508     }
3509     else {
3510       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3511     }
3512   }
3513
3514   /**
3515    * Where/qill clause for greeting fields.
3516    *
3517    * @param array $values
3518    *
3519    * @throws \CRM_Core_Exception
3520    */
3521   public function greetings(&$values) {
3522     list($name, $op, $value, $grouping, $wildcard) = $values;
3523     $name .= '_display';
3524
3525     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3526     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3527     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3528   }
3529
3530   /**
3531    * Where / qill clause for email
3532    *
3533    * @param array $values
3534    * @param string $isForcePrimaryOnly
3535    *
3536    * @throws \CRM_Core_Exception
3537    */
3538   protected function email(&$values, $isForcePrimaryOnly) {
3539     list($name, $op, $value, $grouping, $wildcard) = $values;
3540     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3541
3542     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3543     // so following code is use build whereClause for contact's primart email id
3544     if (!empty($isForcePrimaryOnly)) {
3545       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3546     }
3547     // @todo - this should come from the $this->_fields array
3548     $dbName = $name === 'email_id' ? 'id' : $name;
3549
3550     if (is_array($value) || $name === 'email_id') {
3551       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3552       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3553       return;
3554     }
3555
3556     // Is this ever hit now? Ideally ensure always an array & handle above.
3557     $n = trim($value);
3558     if ($n) {
3559       if (substr($n, 0, 1) == '"' &&
3560         substr($n, -1, 1) == '"'
3561       ) {
3562         $n = substr($n, 1, -1);
3563         $value = CRM_Core_DAO::escapeString($n);
3564         $op = '=';
3565       }
3566       else {
3567         $value = self::getWildCardedValue($wildcard, $op, $n);
3568       }
3569       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3570       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3571     }
3572     else {
3573       $this->_qill[$grouping][] = ts('Email') . " $op ";
3574       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3575     }
3576   }
3577
3578   /**
3579    * Where / qill clause for phone number
3580    *
3581    * @param array $values
3582    *
3583    * @throws \CRM_Core_Exception
3584    */
3585   public function phone_numeric(&$values) {
3586     list($name, $op, $value, $grouping, $wildcard) = $values;
3587     // Strip non-numeric characters; allow wildcards
3588     $number = preg_replace('/[^\d%]/', '', $value);
3589     if ($number) {
3590       if (strpos($number, '%') === FALSE) {
3591         $number = "%$number%";
3592       }
3593
3594       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3595       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3596       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3597     }
3598   }
3599
3600   /**
3601    * Where / qill clause for phone type/location
3602    *
3603    * @param array $values
3604    *
3605    * @throws \CRM_Core_Exception
3606    */
3607   public function phone_option_group($values) {
3608     list($name, $op, $value, $grouping, $wildcard) = $values;
3609     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3610     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3611     $optionName = $options[$value];
3612     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3613     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3614     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3615   }
3616
3617   /**
3618    * Where / qill clause for street_address.
3619    *
3620    * @param array $values
3621    *
3622    * @throws \CRM_Core_Exception
3623    */
3624   public function street_address(&$values) {
3625     list($name, $op, $value, $grouping) = $values;
3626
3627     if (!$op) {
3628       $op = 'LIKE';
3629     }
3630
3631     $n = trim($value);
3632
3633     if ($n) {
3634       if (strpos($value, '%') === FALSE) {
3635         // only add wild card if not there
3636         $value = "%{$value}%";
3637       }
3638       $op = 'LIKE';
3639       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3640       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3641     }
3642     else {
3643       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3644       $this->_qill[$grouping][] = ts('Street') . " $op ";
3645     }
3646
3647     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3648   }
3649
3650   /**
3651    * Where / qill clause for street_unit.
3652    *
3653    * @param array $values
3654    *
3655    * @throws \CRM_Core_Exception
3656    */
3657   public function street_number(&$values) {
3658     list($name, $op, $value, $grouping, $wildcard) = $values;
3659
3660     if (!$op) {
3661       $op = '=';
3662     }
3663
3664     $n = trim($value);
3665
3666     if (strtolower($n) == 'odd') {
3667       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3668       $this->_qill[$grouping][] = ts('Street Number is odd');
3669     }
3670     elseif (strtolower($n) == 'even') {
3671       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3672       $this->_qill[$grouping][] = ts('Street Number is even');
3673     }
3674     else {
3675       $value = $n;
3676       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3677       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3678     }
3679
3680     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3681   }
3682
3683   /**
3684    * Where / qill clause for sorting by character.
3685    *
3686    * @param array $values
3687    */
3688   public function sortByCharacter(&$values) {
3689     list($name, $op, $value, $grouping, $wildcard) = $values;
3690
3691     $name = trim($value);
3692     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3693     $this->_where[$grouping][] = $cond;
3694     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3695   }
3696
3697   /**
3698    * Where / qill clause for including contact ids.
3699    */
3700   public function includeContactIDs() {
3701     if (!$this->_includeContactIds || empty($this->_params)) {
3702       return;
3703     }
3704
3705     $contactIds = [];
3706     foreach ($this->_params as $id => $values) {
3707       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3708         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3709       }
3710     }
3711     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3712     if (!empty($contactIds)) {
3713       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3714     }
3715   }
3716
3717   /**
3718    * Where / qill clause for postal code.
3719    *
3720    * @param array $values
3721    *
3722    * @throws \CRM_Core_Exception
3723    */
3724   public function postalCode(&$values) {
3725     // skip if the fields dont have anything to do with postal_code
3726     if (empty($this->_fields['postal_code'])) {
3727       return;
3728     }
3729
3730     list($name, $op, $value, $grouping, $wildcard) = $values;
3731
3732     // Handle numeric postal code range searches properly by casting the column as numeric
3733     if (is_numeric($value)) {
3734       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3735       $val = CRM_Utils_Type::escape($value, 'Integer');
3736     }
3737     else {
3738       $field = 'civicrm_address.postal_code';
3739       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3740       if (!is_array($value)) {
3741         $val = CRM_Utils_Type::escape($value, 'String');
3742       }
3743       else {
3744         // Do we need to escape values here? I would expect buildClause does.
3745         $val = $value;
3746       }
3747     }
3748
3749     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3750
3751     if ($name == 'postal_code') {
3752       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3753       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3754     }
3755     elseif ($name == 'postal_code_low') {
3756       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3757       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3758     }
3759     elseif ($name == 'postal_code_high') {
3760       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3761       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3762     }
3763   }
3764
3765   /**
3766    * Where / qill clause for location type.
3767    *
3768    * @param array $values
3769    * @param null $status
3770    *
3771    * @return string
3772    */
3773   public function locationType(&$values, $status = NULL) {
3774     list($name, $op, $value, $grouping, $wildcard) = $values;
3775
3776     if (is_array($value)) {
3777       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3778       $this->_tables['civicrm_address'] = 1;
3779       $this->_whereTables['civicrm_address'] = 1;
3780
3781       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3782       $names = [];
3783       foreach ($value as $id) {
3784         $names[] = $locationType[$id];
3785       }
3786
3787       $this->_primaryLocation = FALSE;
3788
3789       if (!$status) {
3790         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3791       }
3792       else {
3793         return implode(' ' . ts('or') . ' ', $names);
3794       }
3795     }
3796   }
3797
3798   /**
3799    * @param $values
3800    * @param bool $fromStateProvince
3801    *
3802    * @return array|NULL
3803    * @throws \CRM_Core_Exception
3804    */
3805   public function country(&$values, $fromStateProvince = TRUE) {
3806     list($name, $op, $value, $grouping, $wildcard) = $values;
3807
3808     if (!$fromStateProvince) {
3809       $stateValues = $this->getWhereValues('state_province', $grouping);
3810       if (!empty($stateValues)) {
3811         // return back to caller if there are state province values
3812         // since that handles this case
3813         return NULL;
3814       }
3815     }
3816
3817     $countryClause = $countryQill = NULL;
3818     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3819       $this->_tables['civicrm_address'] = 1;
3820       $this->_whereTables['civicrm_address'] = 1;
3821
3822       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3823       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3824       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3825
3826       if (!$fromStateProvince) {
3827         $this->_where[$grouping][] = $countryClause;
3828         $this->_qill[$grouping][] = $countryQill;
3829       }
3830     }
3831
3832     if ($fromStateProvince) {
3833       if (!empty($countryClause)) {
3834         return [
3835           $countryClause,
3836           " ...AND... " . $countryQill,
3837         ];
3838       }
3839       else {
3840         return [NULL, NULL];
3841       }
3842     }
3843   }
3844
3845   /**
3846    * Where / qill clause for county (if present).
3847    *
3848    * @param array $values
3849    * @param null $status
3850    *
3851    * @return string
3852    */
3853   public function county(&$values, $status = NULL) {
3854     list($name, $op, $value, $grouping, $wildcard) = $values;
3855
3856     if (!is_array($value)) {
3857       // force the county to be an array
3858       $value = [$value];
3859     }
3860
3861     // check if the values are ids OR names of the counties
3862     $inputFormat = 'id';
3863     foreach ($value as $v) {
3864       if (!is_numeric($v)) {
3865         $inputFormat = 'name';
3866         break;
3867       }
3868     }
3869     $names = [];
3870     if ($op == '=') {
3871       $op = 'IN';
3872     }
3873     elseif ($op == '!=') {
3874       $op = 'NOT IN';
3875     }
3876     else {
3877       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3878       $op = str_replace('EMPTY', 'NULL', $op);
3879     }
3880
3881     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3882       $clause = "civicrm_address.county_id $op";
3883     }
3884     elseif ($inputFormat == 'id') {
3885       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3886
3887       $county = CRM_Core_PseudoConstant::county();
3888       foreach ($value as $id) {
3889         $names[] = $county[$id] ?? NULL;
3890       }
3891     }
3892     else {
3893       $inputClause = [];
3894       $county = CRM_Core_PseudoConstant::county();
3895       foreach ($value as $name) {
3896         $name = trim($name);
3897         $inputClause[] = CRM_Utils_Array::key($name, $county);
3898       }
3899       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3900       $names = $value;
3901     }
3902     $this->_tables['civicrm_address'] = 1;
3903     $this->_whereTables['civicrm_address'] = 1;
3904
3905     $this->_where[$grouping][] = $clause;
3906     if (!$status) {
3907       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3908     }
3909     else {
3910       return implode(' ' . ts('or') . ' ', $names);
3911     }
3912   }
3913
3914   /**
3915    * Where / qill clause for state/province AND country (if present).
3916    *
3917    * @param array $values
3918    * @param null $status
3919    *
3920    * @return string
3921    * @throws \CRM_Core_Exception
3922    */
3923   public function stateProvince(&$values, $status = NULL) {
3924     list($name, $op, $value, $grouping, $wildcard) = $values;
3925
3926     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3927     $this->_tables['civicrm_address'] = 1;
3928     $this->_whereTables['civicrm_address'] = 1;
3929
3930     $countryValues = $this->getWhereValues('country', $grouping);
3931     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3932     if ($countryClause) {
3933       $clause = "( $stateClause AND $countryClause )";
3934     }
3935     else {
3936       $clause = $stateClause;
3937     }
3938
3939     $this->_where[$grouping][] = $clause;
3940     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3941     if (!$status) {
3942       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3943     }
3944     else {
3945       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3946     }
3947   }
3948
3949   /**
3950    * Where / qill clause for change log.
3951    *
3952    * @param array $values
3953    */
3954   public function changeLog(&$values) {
3955     list($name, $op, $value, $grouping, $wildcard) = $values;
3956
3957     $targetName = $this->getWhereValues('changed_by', $grouping);
3958     if (!$targetName) {
3959       return;
3960     }
3961
3962     $name = trim($targetName[2]);
3963     $name = CRM_Core_DAO::escapeString($name);
3964     $name = $targetName[4] ? "%$name%" : $name;
3965     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3966     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3967     $fieldTitle = ts('Altered By');
3968
3969     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3970     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3971       1 => $fieldTitle,
3972       2 => $qillop,
3973       3 => $qillVal,
3974     ]);
3975   }
3976
3977   /**
3978    * @param $values
3979    *
3980    * @throws \CRM_Core_Exception
3981    */
3982   public function modifiedDates($values) {
3983     $this->_useDistinct = TRUE;
3984
3985     // CRM-11281, default to added date if not set
3986     $fieldTitle = ts('Added Date');
3987     $fieldName = 'created_date';
3988     foreach (array_keys($this->_params) as $id) {
3989       if ($this->_params[$id][0] == 'log_date') {
3990         if ($this->_params[$id][2] == 2) {
3991           $fieldTitle = ts('Modified Date');
3992           $fieldName = 'modified_date';
3993         }
3994       }
3995     }
3996
3997     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
3998
3999     self::$_openedPanes[ts('Change Log')] = TRUE;
4000   }
4001
4002   /**
4003    * @param $values
4004    *
4005    * @throws \CRM_Core_Exception
4006    */
4007   public function demographics(&$values) {
4008     list($name, $op, $value, $grouping, $wildcard) = $values;
4009
4010     if (($name == 'age_low') || ($name == 'age_high')) {
4011       $this->ageRangeQueryBuilder($values,
4012         'contact_a', 'age', 'birth_date', ts('Age')
4013       );
4014     }
4015     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4016
4017       $this->dateQueryBuilder($values,
4018         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4019       );
4020     }
4021     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4022
4023       $this->dateQueryBuilder($values,
4024         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4025       );
4026     }
4027
4028     self::$_openedPanes[ts('Demographics')] = TRUE;
4029   }
4030
4031   /**
4032    * @param $values
4033    */
4034   public function privacy(&$values) {
4035     list($name, $op, $value, $grouping) = $values;
4036     if (is_array($value)) {
4037       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4038         $op = key($value);
4039         $value = $value[$op];
4040       }
4041     }
4042     $field = $this->_fields[$name] ?? NULL;
4043     CRM_Utils_Type::validate($value, 'Integer');
4044     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4045     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4046     $title = $field ? $field['title'] : $name;
4047     $this->_qill[$grouping][] = "$title $op $value";
4048   }
4049
4050   /**
4051    * @param $values
4052    */
4053   public function privacyOptions($values) {
4054     list($name, $op, $value, $grouping, $wildcard) = $values;
4055
4056     if (empty($value) || !is_array($value)) {
4057       return;
4058     }
4059
4060     // get the operator and toggle values
4061     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4062     $operator = 'OR';
4063     if ($opValues &&
4064       strtolower($opValues[2] == 'AND')
4065     ) {
4066       // @todo this line is logially unreachable
4067       $operator = 'AND';
4068     }
4069
4070     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4071     $compareOP = '!';
4072     if ($toggleValues &&
4073       $toggleValues[2] == 2
4074     ) {
4075       $compareOP = '';
4076     }
4077
4078     $clauses = [];
4079     $qill = [];
4080     foreach ($value as $dontCare => $pOption) {
4081       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4082       $field = $this->_fields[$pOption] ?? NULL;
4083       $title = $field ? $field['title'] : $pOption;
4084       $qill[] = " $title = 1 ";
4085     }
4086
4087     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4088     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4089   }
4090
4091   /**
4092    * @param $values
4093    *
4094    * @throws \CRM_Core_Exception
4095    */
4096   public function preferredCommunication(&$values) {
4097     list($name, $op, $value, $grouping, $wildcard) = $values;
4098
4099     if (!is_array($value)) {
4100       $value = str_replace(['(', ')'], '', explode(",", $value));
4101     }
4102     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4103       $op = key($value);
4104       $value = $value[$op];
4105     }
4106     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4107
4108     if (self::caseImportant($op)) {
4109       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4110       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4111       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4112     }
4113
4114     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4115     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4116   }
4117
4118   /**
4119    * Where / qill clause for relationship.
4120    *
4121    * @param array $values
4122    */
4123   public function relationship(&$values) {
4124     list($name, $op, $value, $grouping, $wildcard) = $values;
4125     if ($this->_relationshipValuesAdded) {
4126       return;
4127     }
4128     // also get values array for relation_target_name
4129     // for relationship search we always do wildcard
4130     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4131     $description = $this->getWhereValues('relation_description', $grouping);
4132     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4133     $relStatus = $this->getWhereValues('relation_status', $grouping);
4134     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4135
4136     $nameClause = $name = NULL;
4137     if ($targetName) {
4138       $name = trim($targetName[2]);
4139       if (substr($name, 0, 1) == '"' &&
4140         substr($name, -1, 1) == '"'
4141       ) {
4142         $name = substr($name, 1, -1);
4143         $name = CRM_Core_DAO::escapeString($name);
4144         $nameClause = "= '$name'";
4145       }
4146       else {
4147         $name = CRM_Core_DAO::escapeString($name);
4148         $nameClause = "LIKE '%{$name}%'";
4149       }
4150     }
4151
4152     $relTypes = $relTypesIds = [];
4153     if (!empty($relationType)) {
4154       $relationType[2] = (array) $relationType[2];
4155       foreach ($relationType[2] as $relType) {
4156         $rel = explode('_', $relType);
4157         self::$_relType = $rel[1];
4158         $params = ['id' => $rel[0]];
4159         $typeValues = [];
4160         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4161         if (!empty($rTypeValue)) {
4162           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4163             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4164             self::$_relType = 'reciprocal';
4165           }
4166           $relTypesIds[] = $rel[0];
4167           $relTypes[] = $relType;
4168         }
4169       }
4170     }
4171
4172     // if we are creating a temp table we build our own where for the relationship table
4173     $relationshipTempTable = NULL;
4174     if (self::$_relType == 'reciprocal') {
4175       $where = [];
4176       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4177         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4178         ->getName();
4179       if ($nameClause) {
4180         $where[$grouping][] = " sort_name $nameClause ";
4181       }
4182       $groupJoinTable = "civicrm_relationship";
4183       $groupJoinColumn = "contact_id_alt";
4184     }
4185     else {
4186       $where = &$this->_where;
4187       if ($nameClause) {
4188         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4189       }
4190       $groupJoinTable = "contact_b";
4191       $groupJoinColumn = "id";
4192     }
4193     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4194     if ($nameClause || !$targetGroup) {
4195       if (!empty($relationType)) {
4196         $relQill = '';
4197         foreach ($relTypes as $rel) {
4198           if (!empty($relQill)) {
4199             $relQill .= ' OR ';
4200           }
4201           $relQill .= $allRelationshipType[$rel];
4202         }
4203         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4204       }
4205       elseif ($name) {
4206         $this->_qill[$grouping][] = $name;
4207       }
4208     }
4209
4210     //check to see if the target contact is in specified group
4211     if ($targetGroup) {
4212       //add contacts from static groups
4213       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4214         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4215       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4216         implode(",", $targetGroup[2]) . ") ) ";
4217
4218       //add contacts from saved searches
4219       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4220
4221       //set the group where clause
4222       if ($ssWhere) {
4223         $groupWhere[] = "( " . $ssWhere . " )";
4224       }
4225       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4226
4227       //Get the names of the target groups for the qill
4228       $groupNames = CRM_Core_PseudoConstant::group();
4229       $qillNames = [];
4230       foreach ($targetGroup[2] as $groupId) {
4231         if (array_key_exists($groupId, $groupNames)) {
4232           $qillNames[] = $groupNames[$groupId];
4233         }
4234       }
4235       if (!empty($relationType)) {
4236         $relQill = '';
4237         foreach ($relTypes as $rel) {
4238           if (!empty($relQill)) {
4239             $relQill .= ' OR ';
4240           }
4241           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4242         }
4243         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4244       }
4245       else {
4246         $this->_qill[$grouping][] = implode(", ", $qillNames);
4247       }
4248     }
4249
4250     // Description
4251     if (!empty($description[2]) && trim($description[2])) {
4252       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4253       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4254       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4255     }
4256
4257     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4258     $today = date('Ymd');
4259     //check for active, inactive and all relation status
4260     if (empty($relStatus[2])) {
4261       $where[$grouping][] = "(
4262 civicrm_relationship.is_active = 1 AND
4263 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4264 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4265 )";
4266       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4267     }
4268     elseif ($relStatus[2] == 1) {
4269       $where[$grouping][] = "(
4270 civicrm_relationship.is_active = 0 OR
4271 civicrm_relationship.end_date < {$today} OR
4272 civicrm_relationship.start_date > {$today}
4273 )";
4274       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4275     }
4276
4277     $onlyDeleted = 0;
4278     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4279       $onlyDeleted = 1;
4280     }
4281     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4282
4283     $this->addRelationshipPermissionClauses($grouping, $where);
4284     $this->addRelationshipDateClauses($grouping, $where);
4285     $this->addRelationshipActivePeriodClauses($grouping, $where);
4286     if (!empty($relTypes)) {
4287       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4288     }
4289     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4290     $this->_useDistinct = TRUE;
4291     $this->_relationshipValuesAdded = TRUE;
4292     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4293     // join on that,
4294     if ($relationshipTempTable) {
4295       $whereClause = '';
4296       if (!empty($where[$grouping])) {
4297         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4298         $whereClause = str_replace('contact_b', 'c', $whereClause);
4299       }
4300       $sql = "
4301         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4302           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4303             FROM civicrm_relationship
4304             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4305             $whereClause )
4306           UNION
4307             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4308             FROM civicrm_relationship
4309             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4310             $whereClause )
4311       ";
4312       CRM_Core_DAO::executeQuery($sql);
4313     }
4314   }
4315
4316   /**
4317    * Add relationship permission criteria to where clause.
4318    *
4319    * @param string $grouping
4320    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4321    *   Not the main query.
4322    */
4323   public function addRelationshipPermissionClauses($grouping, &$where) {
4324     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4325     if ($relPermission) {
4326       if (!is_array($relPermission[2])) {
4327         // this form value was scalar in previous versions of Civi
4328         $relPermission[2] = [$relPermission[2]];
4329       }
4330       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4331
4332       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4333
4334       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4335       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4336     }
4337   }
4338
4339   /**
4340    * Add start & end date criteria in
4341    * @param string $grouping
4342    * @param array $where
4343    *   = array to add where clauses to, in case you are generating a temp table.
4344    * not the main query.
4345    */
4346   public function addRelationshipDateClauses($grouping, &$where) {
4347     foreach (['start_date', 'end_date'] as $dateField) {
4348       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4349       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4350       if (!empty($dateValueLow)) {
4351         $date = date('Ymd', strtotime($dateValueLow[2]));
4352         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4353         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4354       }
4355       if (!empty($dateValueHigh)) {
4356         $date = date('Ymd', strtotime($dateValueHigh[2]));
4357         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4358         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4359       }
4360     }
4361   }
4362
4363   /**
4364    * Add start & end active period criteria in
4365    * @param string $grouping
4366    * @param array $where
4367    *   = array to add where clauses to, in case you are generating a temp table.
4368    * not the main query.
4369    */
4370   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4371     $dateValues = [];
4372     $dateField = 'active_period_date';
4373
4374     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4375     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4376     $dateValueLowFormated = $dateValueHighFormated = NULL;
4377     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4378       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4379       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4380       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4381     }
4382     elseif (!empty($dateValueLow)) {
4383       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4384       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4385     }
4386     elseif (!empty($dateValueHigh)) {
4387       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4388       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4389     }
4390
4391     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4392       $where[$grouping][] = $activePeriodClauses;
4393     }
4394   }
4395
4396   /**
4397    * Get start & end active period criteria
4398    *
4399    * @param $from
4400    * @param $to
4401    * @param $forceTableName
4402    *
4403    * @return string
4404    */
4405   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4406     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4407     if (!is_null($from) && !is_null($to)) {
4408       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4409                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4410                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4411                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4412                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4413                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4414     }
4415     elseif (!is_null($from)) {
4416       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4417                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4418                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4419     }
4420     elseif (!is_null($to)) {
4421       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4422                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4423                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4424     }
4425   }
4426
4427   /**
4428    * Default set of return properties.
4429    *
4430    * @param int $mode
4431    *
4432    * @return array
4433    *   derault return properties
4434    */
4435   public static function &defaultReturnProperties($mode = 1) {
4436     if (!isset(self::$_defaultReturnProperties)) {
4437       self::$_defaultReturnProperties = [];
4438     }
4439
4440     if (!isset(self::$_defaultReturnProperties[$mode])) {
4441       // add activity return properties
4442       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4443         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4444       }
4445       else {
4446         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4447       }
4448
4449       if (empty(self::$_defaultReturnProperties[$mode])) {
4450         self::$_defaultReturnProperties[$mode] = [
4451           'home_URL' => 1,
4452           'image_URL' => 1,
4453           'legal_identifier' => 1,
4454           'external_identifier' => 1,
4455           'contact_type' => 1,
4456           'contact_sub_type' => 1,
4457           'sort_name' => 1,
4458           'display_name' => 1,
4459           'preferred_mail_format' => 1,
4460           'nick_name' => 1,
4461           'first_name' => 1,
4462           'middle_name' => 1,
4463           'last_name' => 1,
4464           'prefix_id' => 1,
4465           'suffix_id' => 1,
4466           'formal_title' => 1,
4467           'communication_style_id' => 1,
4468           'birth_date' => 1,
4469           'gender_id' => 1,
4470           'street_address' => 1,
4471           'supplemental_address_1' => 1,
4472           'supplemental_address_2' => 1,
4473           'supplemental_address_3' => 1,
4474           'city' => 1,
4475           'postal_code' => 1,
4476           'postal_code_suffix' => 1,
4477           'state_province' => 1,
4478           'country' => 1,
4479           'world_region' => 1,
4480           'geo_code_1' => 1,
4481           'geo_code_2' => 1,
4482           'email' => 1,
4483           'on_hold' => 1,
4484           'phone' => 1,
4485           'im' => 1,
4486           'household_name' => 1,
4487           'organization_name' => 1,
4488           'deceased_date' => 1,
4489           'is_deceased' => 1,
4490           'job_title' => 1,
4491           'legal_name' => 1,
4492           'sic_code' => 1,
4493           'current_employer' => 1,
4494           // FIXME: should we use defaultHierReturnProperties() for the below?
4495           'do_not_email' => 1,
4496           'do_not_mail' => 1,
4497           'do_not_sms' => 1,
4498           'do_not_phone' => 1,
4499           'do_not_trade' => 1,
4500           'is_opt_out' => 1,
4501           'contact_is_deleted' => 1,
4502           'preferred_communication_method' => 1,
4503           'preferred_language' => 1,
4504         ];
4505       }
4506     }
4507     return self::$_defaultReturnProperties[$mode];
4508   }
4509
4510   /**
4511    * Get primary condition for a sql clause.
4512    *
4513    * @param int $value
4514    *
4515    * @return string|NULL
4516    */
4517   public static function getPrimaryCondition($value) {
4518     if (is_numeric($value)) {
4519       $value = (int ) $value;
4520       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4521     }
4522     return NULL;
4523   }
4524
4525   /**
4526    * Wrapper for a simple search query.
4527    *
4528    * @param array $params
4529    * @param array $returnProperties
4530    * @param bool $count
4531    *
4532    * @return string
4533    * @throws \CRM_Core_Exception
4534    */
4535   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4536     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4537     list($select, $from, $where, $having) = $query->query();
4538     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4539
4540     $query = "$select $from $where $groupBy $having";
4541     return $query;
4542   }
4543
4544   /**
4545    * These are stub comments as this function needs more explanation - particularly in terms of how it
4546    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4547    *
4548    * This function was originally written as a wrapper for the api query but is called from multiple places
4549    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4550    * but it is unclear as to whehter that is historical or there is a reason
4551    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4552    *
4553    * @param array $params
4554    * @param array $returnProperties
4555    * @param null $fields
4556    * @param string $sort
4557    * @param int $offset
4558    * @param int $row_count
4559    * @param bool $smartGroupCache
4560    *   ?? update smart group cache?.
4561    * @param bool $count
4562    *   Return count obnly.
4563    * @param bool $skipPermissions
4564    *   Should permissions be ignored or should the logged in user's permissions be applied.
4565    * @param int $mode
4566    *   This basically correlates to the component.
4567    * @param string $apiEntity
4568    *   The api entity being called.
4569    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4570    *
4571    * @param bool|null $primaryLocationOnly
4572    *
4573    * @return array
4574    * @throws \CRM_Core_Exception
4575    */
4576   public static function apiQuery(
4577     $params = NULL,
4578     $returnProperties = NULL,
4579     $fields = NULL,
4580     $sort = NULL,
4581     $offset = 0,
4582     $row_count = 25,
4583     $smartGroupCache = TRUE,
4584     $count = FALSE,
4585     $skipPermissions = TRUE,
4586     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4587     $apiEntity = NULL,
4588     $primaryLocationOnly = NULL
4589   ) {
4590
4591     $query = new CRM_Contact_BAO_Query(
4592       $params, $returnProperties,
4593       NULL, TRUE, FALSE, $mode,
4594       $skipPermissions,
4595       TRUE, $smartGroupCache,
4596       NULL, 'AND',
4597       $apiEntity, $primaryLocationOnly
4598     );
4599
4600     //this should add a check for view deleted if permissions are enabled
4601     if ($skipPermissions) {
4602       $query->_skipDeleteClause = TRUE;
4603     }
4604     $query->generatePermissionClause(FALSE, $count);
4605
4606     // note : this modifies _fromClause and _simpleFromClause
4607     $query->includePseudoFieldsJoin($sort);
4608
4609     list($select, $from, $where, $having) = $query->query($count);
4610
4611     $options = $query->_options;
4612     if (!empty($query->_permissionWhereClause)) {
4613       if (!empty($query->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
4614         $from .= " $query->_permissionFromClause";
4615       }
4616       if (empty($where)) {
4617         $where = "WHERE $query->_permissionWhereClause";
4618       }
4619       else {
4620         $where = "$where AND $query->_permissionWhereClause";
4621       }
4622     }
4623
4624     $sql = "$select $from $where $having";
4625
4626     // add group by only when API action is not getcount
4627     //  otherwise query fetches incorrect count
4628     if ($query->_useGroupBy && !$count) {
4629       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4630     }
4631     if (!empty($sort)) {
4632       $sort = CRM_Utils_Type::escape($sort, 'String');
4633       $sql .= " ORDER BY $sort ";
4634     }
4635     if ($row_count > 0 && $offset >= 0) {
4636       $offset = CRM_Utils_Type::escape($offset, 'Int');
4637       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4638       $sql .= " LIMIT $offset, $row_count ";
4639     }
4640
4641     $dao = CRM_Core_DAO::executeQuery($sql);
4642
4643     // @todo derive this from the component class rather than hard-code two options.
4644     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4645
4646     $values = [];
4647     while ($dao->fetch()) {
4648       if ($count) {
4649         $noRows = $dao->rowCount;
4650         return [$noRows, NULL];
4651       }
4652       $val = $query->store($dao);
4653       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4654
4655       if (!empty($convertedVals)) {
4656         $val = array_replace_recursive($val, $convertedVals);
4657       }
4658       $values[$dao->$entityIDField] = $val;
4659     }
4660     return [$values, $options];
4661   }
4662
4663   /**
4664    * Get the actual custom field name by stripping off the appended string.
4665    *
4666    * The string could be _relative, _from, or _to
4667    *
4668    * @todo use metadata rather than convention to do this.
4669    *
4670    * @param string $parameterName
4671    *   The name of the parameter submitted to the form.
4672    *   e.g
4673    *   custom_3_relative
4674    *   custom_3_from
4675    *
4676    * @return string
4677    */
4678   public static function getCustomFieldName($parameterName) {
4679     if (substr($parameterName, -5, 5) == '_from') {
4680       return substr($parameterName, 0, strpos($parameterName, '_from'));
4681     }
4682     if (substr($parameterName, -9, 9) == '_relative') {
4683       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4684     }
4685     if (substr($parameterName, -3, 3) == '_to') {
4686       return substr($parameterName, 0, strpos($parameterName, '_to'));
4687     }
4688   }
4689
4690   /**
4691    * Convert submitted values for relative custom fields to query object format.
4692    *
4693    * The query will support the sqlOperator format so convert to that format.
4694    *
4695    * @param array $formValues
4696    *   Submitted values.
4697    * @param array $params
4698    *   Converted parameters for the query object.
4699    * @param string $values
4700    *   Submitted value.
4701    * @param string $fieldName
4702    *   Submitted field name. (Matches form field not DB field.)
4703    */
4704   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4705     if (empty($values)) {
4706       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4707       // this function gets called again after they fields have been converted which can get ugly.
4708       return;
4709     }
4710     $customFieldName = self::getCustomFieldName($fieldName);
4711
4712     if (substr($fieldName, -9, 9) == '_relative') {
4713       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4714     }
4715     else {
4716       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4717         // Both to & from are set. We only need to acton one, choosing from.
4718         return;
4719       }
4720
4721       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4722       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4723
4724       if (self::isCustomDateField($customFieldName)) {
4725         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4726       }
4727     }
4728
4729     if ($from) {
4730       if ($to) {
4731         $relativeFunction = ['BETWEEN' => [$from, $to]];
4732       }
4733       else {
4734         $relativeFunction = ['>=' => $from];
4735       }
4736     }
4737     else {
4738       $relativeFunction = ['<=' => $to];
4739     }
4740     $params[] = [
4741       $customFieldName,
4742       '=',
4743       $relativeFunction,
4744       0,
4745       0,
4746     ];
4747   }
4748
4749   /**
4750    * Are we dealing with custom field of type date.
4751    *
4752    * @param $fieldName
4753    *
4754    * @return bool
4755    * @throws \CiviCRM_API3_Exception
4756    */
4757   public static function isCustomDateField($fieldName) {
4758     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4759       return FALSE;
4760     }
4761     try {
4762       $customFieldDataType = civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type']);
4763       if ('Date' == $customFieldDataType) {
4764         return TRUE;
4765       }
4766     }
4767     catch (CiviCRM_API3_Exception $e) {
4768     }
4769     return FALSE;
4770   }
4771
4772   /**
4773    * Has this field already been reformatting to Query object syntax.
4774    *
4775    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4776    * to come with associated double queries & is possibly damaging performance.
4777    *
4778    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4779    * them as they are.
4780    *
4781    * @param mixed $values
4782    *   Value in formValues for the field.
4783    *
4784    * @return bool;
4785    */
4786   public static function isAlreadyProcessedForQueryFormat($values) {
4787     if (!is_array($values)) {
4788       return FALSE;
4789     }
4790     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4791       return FALSE;
4792     }
4793     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4794   }
4795
4796   /**
4797    * If the state and country are passed remove state.
4798    *
4799    * Country is implicit from the state, but including both results in
4800    * a poor query as there is no combined index on state AND country.
4801    *
4802    * @see https://issues.civicrm.org/jira/browse/CRM-18125
4803    *
4804    * @param array $formValues
4805    */
4806   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4807     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4808       // The use of array map sanitises the data by ensuring we are dealing with integers.
4809       $states = implode(', ', array_map('intval', $formValues['state_province']));
4810       $countryList = CRM_Core_DAO::singleValueQuery(
4811         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4812       );
4813       if ($countryList == $formValues['country']) {
4814         unset($formValues['country']);
4815       }
4816     }
4817   }
4818
4819   /**
4820    * For some special cases, grouping by subset of select fields becomes mandatory.
4821    * Hence, full_group_by mode is handled by appending any_value
4822    * keyword to select fields not present in groupBy
4823    *
4824    * @param array $selectClauses
4825    * @param array $groupBy - Columns already included in GROUP By clause.
4826    * @param string $aggregateFunction
4827    *
4828    * @return string
4829    */
4830   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4831     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4832       $groupBy = array_map('trim', (array) $groupBy);
4833       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4834       foreach ($selectClauses as $key => &$val) {
4835         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4836         // append ANY_VALUE() keyword
4837         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4838           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4839             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4840             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4841         }
4842       }
4843     }
4844
4845     return "SELECT " . implode(', ', $selectClauses) . " ";
4846   }
4847
4848   /**
4849    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4850    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4851    * keyword to select fields not present in groupBy
4852    *
4853    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4854    * @param array $orderBys - ORDER BY sub-clauses
4855    *
4856    */
4857   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4858     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4859       foreach ($orderBys as $orderBy) {
4860         // remove sort syntax from ORDER BY clauses if present
4861         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4862         // if ORDER BY column is not present in GROUP BY then append it to end
4863         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4864           $groupBy .= ", {$orderBy}";
4865         }
4866       }
4867     }
4868   }
4869
4870   /**
4871    * Include Select columns in groupBy clause.
4872    *
4873    * @param array $selectClauses
4874    * @param array $groupBy - Columns already included in GROUP By clause.
4875    *
4876    * @return string
4877    */
4878   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4879     $groupBy = (array) $groupBy;
4880     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4881     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4882
4883     //return if ONLY_FULL_GROUP_BY is not enabled.
4884     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4885       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4886       foreach ($selectClauses as $key => $val) {
4887         $aliasArray = preg_split('/ as /i', $val);
4888         // if more than 1 alias we need to split by ','.
4889         if (count($aliasArray) > 2) {
4890           $aliasArray = preg_split('/,/', $val);
4891           foreach ($aliasArray as $key => $value) {
4892             $alias = current(preg_split('/ as /i', $value));
4893             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4894               $groupBy[] = $alias;
4895             }
4896           }
4897         }
4898         else {
4899           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4900           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4901           $tableName = current(explode('.', $selectColumn));
4902           $primaryKey = "{$tableName}.id";
4903           // exclude columns which are already included in groupBy and aggregate functions from select
4904           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4905           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4906             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4907               $groupBy[] = $alias;
4908             }
4909             else {
4910               $groupBy[] = $selectColumn;
4911             }
4912           }
4913         }
4914       }
4915     }
4916
4917     if (!empty($groupBy)) {
4918       return " GROUP BY " . implode(', ', $groupBy);
4919     }
4920     return '';
4921   }
4922
4923   /**
4924    * Create and query the db for an contact search.
4925    *
4926    * @param int $offset
4927    *   The offset for the query.
4928    * @param int $rowCount
4929    *   The number of rows to return.
4930    * @param string|CRM_Utils_Sort $sort
4931    *   The order by string.
4932    * @param bool $count
4933    *   Is this a count only query ?.
4934    * @param bool $includeContactIds
4935    *   Should we include contact ids?.
4936    * @param bool $sortByChar
4937    *   If true returns the distinct array of first characters for search results.
4938    * @param bool $groupContacts
4939    *   If true, return only the contact ids.
4940    * @param bool $returnQuery
4941    *   Should we return the query as a string.
4942    * @param string $additionalWhereClause
4943    *   If the caller wants to further restrict the search (used for components).
4944    * @param null $sortOrder
4945    * @param string $additionalFromClause
4946    *   Should be clause with proper joins, effective to reduce where clause load.
4947    *
4948    * @param bool $skipOrderAndLimit
4949    *
4950    * @return CRM_Core_DAO
4951    */
4952   public function searchQuery(
4953     $offset = 0, $rowCount = 0, $sort = NULL,
4954     $count = FALSE, $includeContactIds = FALSE,
4955     $sortByChar = FALSE, $groupContacts = FALSE,
4956     $returnQuery = FALSE,
4957     $additionalWhereClause = NULL, $sortOrder = NULL,
4958     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4959   ) {
4960
4961     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4962
4963     if ($returnQuery) {
4964       return $query;
4965     }
4966     if ($count) {
4967       return CRM_Core_DAO::singleValueQuery($query);
4968     }
4969
4970     $dao = CRM_Core_DAO::executeQuery($query);
4971
4972     // We can always call this - it will only re-enable if it was originally enabled.
4973     CRM_Core_DAO::reenableFullGroupByMode();
4974
4975     if ($groupContacts) {
4976       $ids = [];
4977       while ($dao->fetch()) {
4978         $ids[] = $dao->id;
4979       }
4980       return implode(',', $ids);
4981     }
4982
4983     return $dao;
4984   }
4985
4986   /**
4987    * Create and query the db for the list of all first letters used by contacts
4988    *
4989    * @return CRM_Core_DAO
4990    */
4991   public function alphabetQuery() {
4992     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
4993     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
4994       {$sqlParts['from']}
4995       {$sqlParts['where']}";
4996     $dao = CRM_Core_DAO::executeQuery($query);
4997     return $dao;
4998   }
4999
5000   /**
5001    * Fetch a list of contacts for displaying a search results page
5002    *
5003    * @param array $cids
5004    *   List of contact IDs
5005    * @param bool $includeContactIds
5006    * @return CRM_Core_DAO
5007    */
5008   public function getCachedContacts($cids, $includeContactIds) {
5009     CRM_Core_DAO::disableFullGroupByMode();
5010     CRM_Utils_Type::validateAll($cids, 'Positive');
5011     $this->_includeContactIds = $includeContactIds;
5012     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5013     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5014     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5015     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5016     $order = 'ORDER BY _wgt';
5017     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5018     $limit = '';
5019     $query = "$select $from $where $groupBy $order $limit";
5020
5021     $result = CRM_Core_DAO::executeQuery($query);
5022     CRM_Core_DAO::reenableFullGroupByMode();
5023     return $result;
5024   }
5025
5026   /**
5027    * Construct a SQL CASE expression.
5028    *
5029    * @param string $idCol
5030    *   The name of a column with ID's (eg 'contact_a.id').
5031    * @param array $cids
5032    *   Array(int $weight => int $id).
5033    * @return string
5034    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5035    */
5036   private function createSqlCase($idCol, $cids) {
5037     $buf = "CASE\n";
5038     foreach ($cids as $weight => $cid) {
5039       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5040     }
5041     $buf .= "END\n";
5042     return $buf;
5043   }
5044
5045   /**
5046    * Populate $this->_permissionWhereClause with permission related clause and update other
5047    * query related properties.
5048    *
5049    * Function calls ACL permission class and hooks to filter the query appropriately
5050    *
5051    * Note that these 2 params were in the code when extracted from another function
5052    * and a second round extraction would be to make them properties of the class
5053    *
5054    * @param bool $onlyDeleted
5055    *   Only get deleted contacts.
5056    * @param bool $count
5057    *   Return Count only.
5058    */
5059   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5060     if (!$this->_skipPermission) {
5061       $permissionClauses = CRM_Contact_BAO_Contact_Permission::cacheClause();
5062       $this->_permissionWhereClause = $permissionClauses[1];
5063       $this->_permissionFromClause = $permissionClauses[0];
5064
5065       if (CRM_Core_Permission::check('access deleted contacts')) {
5066         if (!$onlyDeleted) {
5067           $this->_permissionWhereClause = str_replace('( 1 )', '(contact_a.is_deleted = 0)', $this->_permissionWhereClause);
5068         }
5069         else {
5070           if ($this->_permissionWhereClause === '( 1 )') {
5071             $this->_permissionWhereClause = str_replace('( 1 )', '(contact_a.is_deleted)', $this->_permissionWhereClause);
5072           }
5073           else {
5074             $this->_permissionWhereClause .= " AND (contact_a.is_deleted) ";
5075           }
5076         }
5077       }
5078
5079       if (isset($this->_tables['civicrm_activity'])) {
5080         $bao = new CRM_Activity_BAO_Activity();
5081         $clauses = $subclauses = [];
5082         foreach ((array) $bao->addSelectWhereClause() as $field => $vals) {
5083           if ($vals && $field !== 'id') {
5084             $clauses[] = $bao->tableName() . ".$field " . $vals;
5085           }
5086           elseif ($vals) {
5087             $subclauses[] = "$field " . implode(" AND $field ", (array) $vals);
5088           }
5089         }
5090         if ($subclauses) {
5091           $clauses[] = $bao->tableName() . '.`id` IN (SELECT `id` FROM `' . $bao->tableName() . '` WHERE ' . implode(' AND ', $subclauses) . ')';
5092         }
5093         if (!empty($clauses) && $this->_permissionWhereClause) {
5094           $this->_permissionWhereClause .= ' AND (' . implode(' AND ', $clauses) . ')';
5095         }
5096         elseif (!empty($clauses)) {
5097           $this->_permissionWhereClause .= '(' . implode(' AND ', $clauses) . ')';
5098         }
5099       }
5100     }
5101     else {
5102       // add delete clause if needed even if we are skipping permission
5103       // CRM-7639
5104       if (!$this->_skipDeleteClause) {
5105         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5106           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5107         }
5108         else {
5109           // CRM-6181
5110           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5111         }
5112       }
5113     }
5114   }
5115
5116   /**
5117    * @param $val
5118    */
5119   public function setSkipPermission($val) {
5120     $this->_skipPermission = $val;
5121   }
5122
5123   /**
5124    * @param null $context
5125    *
5126    * @return array
5127    * @throws \CRM_Core_Exception
5128    */
5129   public function summaryContribution($context = NULL) {
5130     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5131     if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5132       $from .= " $this->_permissionFromClause";
5133     }
5134     if ($this->_permissionWhereClause) {
5135       $where .= " AND " . $this->_permissionWhereClause;
5136     }
5137     if ($context == 'search') {
5138       $where .= " AND contact_a.is_deleted = 0 ";
5139     }
5140
5141     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5142
5143     $summary = ['total' => []];
5144     $this->addBasicStatsToSummary($summary, $where, $from);
5145
5146     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5147       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5148     }
5149
5150     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5151
5152     return $summary;
5153   }
5154
5155   /**
5156    * Append financial ACL limits to the query from & where clauses, if applicable.
5157    *
5158    * @param string $where
5159    * @param string $from
5160    */
5161   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5162     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5163       return;
5164     }
5165     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5166     if (!empty($financialTypes)) {
5167       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5168       $from .= " LEFT JOIN civicrm_line_item li
5169                   ON civicrm_contribution.id = li.contribution_id AND
5170                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5171     }
5172     else {
5173       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5174     }
5175   }
5176
5177   /**
5178    * Getter for the qill object.
5179    *
5180    * @return array
5181    */
5182   public function qill() {
5183     return $this->_qill;
5184   }
5185
5186   /**
5187    * Default set of return default hier return properties.
5188    *
5189    * @return array
5190    */
5191   public static function &defaultHierReturnProperties() {
5192     if (!isset(self::$_defaultHierReturnProperties)) {
5193       self::$_defaultHierReturnProperties = [
5194         'home_URL' => 1,
5195         'image_URL' => 1,
5196         'legal_identifier' => 1,
5197         'external_identifier' => 1,
5198         'contact_type' => 1,
5199         'contact_sub_type' => 1,
5200         'sort_name' => 1,
5201         'display_name' => 1,
5202         'nick_name' => 1,
5203         'first_name' => 1,
5204         'middle_name' => 1,
5205         'last_name' => 1,
5206         'prefix_id' => 1,
5207         'suffix_id' => 1,
5208         'formal_title' => 1,
5209         'communication_style_id' => 1,
5210         'email_greeting' => 1,
5211         'postal_greeting' => 1,
5212         'addressee' => 1,
5213         'birth_date' => 1,
5214         'gender_id' => 1,
5215         'preferred_communication_method' => 1,
5216         'do_not_phone' => 1,
5217         'do_not_email' => 1,
5218         'do_not_mail' => 1,
5219         'do_not_sms' => 1,
5220         'do_not_trade' => 1,
5221         'location' => [
5222           '1' => [
5223             'location_type' => 1,
5224             'street_address' => 1,
5225             'city' => 1,
5226             'state_province' => 1,
5227             'postal_code' => 1,
5228             'postal_code_suffix' => 1,
5229             'country' => 1,
5230             'phone-Phone' => 1,
5231             'phone-Mobile' => 1,
5232             'phone-Fax' => 1,
5233             'phone-1' => 1,
5234             'phone-2' => 1,
5235             'phone-3' => 1,
5236             'im-1' => 1,
5237             'im-2' => 1,
5238             'im-3' => 1,
5239             'email-1' => 1,
5240             'email-2' => 1,
5241             'email-3' => 1,
5242           ],
5243           '2' => [
5244             'location_type' => 1,
5245             'street_address' => 1,
5246             'city' => 1,
5247             'state_province' => 1,
5248             'postal_code' => 1,
5249             'postal_code_suffix' => 1,
5250             'country' => 1,
5251             'phone-Phone' => 1,
5252             'phone-Mobile' => 1,
5253             'phone-1' => 1,
5254             'phone-2' => 1,
5255             'phone-3' => 1,
5256             'im-1' => 1,
5257             'im-2' => 1,
5258             'im-3' => 1,
5259             'email-1' => 1,
5260             'email-2' => 1,
5261             'email-3' => 1,
5262           ],
5263         ],
5264       ];
5265     }
5266     return self::$_defaultHierReturnProperties;
5267   }
5268
5269   /**
5270    * Build query for a date field.
5271    *
5272    * @param array $values
5273    * @param string $tableName
5274    * @param string $fieldName
5275    * @param string $dbFieldName
5276    * @param string $fieldTitle
5277    * @param bool $appendTimeStamp
5278    * @param string $dateFormat
5279    * @param string|null $highDBFieldName
5280    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5281    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5282    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5283    *   date & less than the end date.
5284    *
5285    * @throws \CRM_Core_Exception
5286    */
5287   public function dateQueryBuilder(
5288     $values, $tableName, $fieldName,
5289     $dbFieldName, $fieldTitle,
5290     $appendTimeStamp = TRUE,
5291     $dateFormat = 'YmdHis',
5292     $highDBFieldName = NULL
5293   ) {
5294     // @todo - remove dateFormat - pretty sure it's never passed in...
5295     list($name, $op, $value, $grouping, $wildcard) = $values;
5296     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5297       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5298       return;
5299     }
5300     if ($tableName === 'civicrm_contact') {
5301       // Special handling for contact table as it has a known alias in advanced search.
5302       $tableName = 'contact_a';
5303     }
5304     if ($name === "{$fieldName}_low" ||
5305       $name === "{$fieldName}_high"
5306     ) {
5307       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5308         return;
5309       }
5310       $this->_rangeCache[$fieldName] = 1;
5311
5312       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5313
5314       if ($name == $fieldName . '_low') {
5315         $firstOP = '>=';
5316         $firstPhrase = ts('greater than or equal to');
5317         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5318
5319         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5320         if (!empty($secondValues) && $secondValues[2]) {
5321           $secondOP = '<=';
5322           $secondPhrase = ts('less than or equal to');
5323           $secondValue = $secondValues[2];
5324
5325           if ($appendTimeStamp && strlen($secondValue) == 10) {
5326             $secondValue .= ' 23:59:59';
5327           }
5328           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5329         }
5330       }
5331       elseif ($name == $fieldName . '_high') {
5332         $firstOP = '<=';
5333         $firstPhrase = ts('less than or equal to');
5334
5335         if ($appendTimeStamp && strlen($value) == 10) {
5336           $value .= ' 23:59:59';
5337         }
5338         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5339
5340         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5341         if (!empty($secondValues) && $secondValues[2]) {
5342           $secondOP = '>=';
5343           $secondPhrase = ts('greater than or equal to');
5344           $secondValue = $secondValues[2];
5345           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5346         }
5347       }
5348
5349       if (!$appendTimeStamp) {
5350         $firstDate = substr($firstDate, 0, 8);
5351       }
5352       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5353
5354       if ($secondDate) {
5355         if (!$appendTimeStamp) {
5356           $secondDate = substr($secondDate, 0, 8);
5357         }
5358         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5359       }
5360
5361       if ($secondDate) {
5362         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5363         $this->_where[$grouping][] = "
5364 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5365 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5366 ";
5367         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5368       }
5369       else {
5370         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5371         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5372       }
5373     }
5374
5375     if ($name == $fieldName) {
5376       //In Get API, for operators other then '=' the $value is in array(op => value) format
5377       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5378         $op = key($value);
5379         $value = $value[$op];
5380       }
5381
5382       $date = $format = NULL;
5383       if (strstr($op, 'IN')) {
5384         $format = [];
5385         foreach ($value as &$date) {
5386           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5387           if (!$appendTimeStamp) {
5388             $date = substr($date, 0, 8);
5389           }
5390           $format[] = CRM_Utils_Date::customFormat($date);
5391         }
5392         $date = "('" . implode("','", $value) . "')";
5393         $format = implode(', ', $format);
5394       }
5395       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5396         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5397         if (!$appendTimeStamp) {
5398           $date = substr($date, 0, 8);
5399         }
5400         $format = CRM_Utils_Date::customFormat($date);
5401         $date = "'$date'";
5402       }
5403
5404       if ($date) {
5405         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5406       }
5407       else {
5408         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5409       }
5410
5411       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5412       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5413     }
5414
5415     // Ensure the tables are set, but don't whomp anything.
5416     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5417     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5418   }
5419
5420   /**
5421    * @param $values
5422    * @param string $tableName
5423    * @param string $fieldName
5424    * @param string $dbFieldName
5425    * @param $fieldTitle
5426    * @param null $options
5427    */
5428   public function numberRangeBuilder(
5429     &$values,
5430     $tableName, $fieldName,
5431     $dbFieldName, $fieldTitle,
5432     $options = NULL
5433   ) {
5434     list($name, $op, $value, $grouping, $wildcard) = $values;
5435
5436     if ($name == "{$fieldName}_low" ||
5437       $name == "{$fieldName}_high"
5438     ) {
5439       if (isset($this->_rangeCache[$fieldName])) {
5440         return;
5441       }
5442       $this->_rangeCache[$fieldName] = 1;
5443
5444       $secondOP = $secondPhrase = $secondValue = NULL;
5445
5446       if ($name == "{$fieldName}_low") {
5447         $firstOP = '>=';
5448         $firstPhrase = ts('greater than');
5449
5450         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5451         if (!empty($secondValues)) {
5452           $secondOP = '<=';
5453           $secondPhrase = ts('less than');
5454           $secondValue = $secondValues[2];
5455         }
5456       }
5457       else {
5458         $firstOP = '<=';
5459         $firstPhrase = ts('less than');
5460
5461         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5462         if (!empty($secondValues)) {
5463           $secondOP = '>=';
5464           $secondPhrase = ts('greater than');
5465           $secondValue = $secondValues[2];
5466         }
5467       }
5468
5469       if ($secondOP) {
5470         $this->_where[$grouping][] = "
5471 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5472 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5473 ";
5474         $displayValue = $options ? $options[$value] : $value;
5475         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5476
5477         $this->_qill[$grouping][]
5478           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5479       }
5480       else {
5481         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5482         $displayValue = $options ? $options[$value] : $value;
5483         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5484       }
5485       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5486
5487       return;
5488     }
5489
5490     if ($name == $fieldName) {
5491       $op = '=';
5492       $phrase = '=';
5493
5494       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5495
5496       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5497       $displayValue = $options ? $options[$value] : $value;
5498       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5499     }
5500   }
5501
5502   /**
5503    * @param $values
5504    * @param string $tableName
5505    * @param string $fieldName
5506    * @param string $dbFieldName
5507    * @param $fieldTitle
5508    * @param null $options
5509    */
5510   public function ageRangeQueryBuilder(
5511     &$values,
5512     $tableName, $fieldName,
5513     $dbFieldName, $fieldTitle,
5514     $options = NULL
5515   ) {
5516     list($name, $op, $value, $grouping, $wildcard) = $values;
5517
5518     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5519     // will be treated as current day
5520     $asofDate = NULL;
5521     if ($asofDateValues) {
5522       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5523       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5524       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5525     }
5526
5527     if ($name == "{$fieldName}_low" ||
5528       $name == "{$fieldName}_high"
5529     ) {
5530       if (isset($this->_rangeCache[$fieldName])) {
5531         return;
5532       }
5533       $this->_rangeCache[$fieldName] = 1;
5534
5535       $secondOP = $secondPhrase = $secondValue = NULL;
5536
5537       if ($name == "{$fieldName}_low") {
5538         $firstPhrase = ts('greater than or equal to');
5539         // NB: age > X means date of birth < Y
5540         $firstOP = '<=';
5541         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5542
5543         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5544         if (!empty($secondValues)) {
5545           $secondOP = '>=';
5546           $secondPhrase = ts('less than or equal to');
5547           $secondValue = $secondValues[2];
5548           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5549         }
5550       }
5551       else {
5552         $firstOP = '>=';
5553         $firstPhrase = ts('less than or equal to');
5554         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5555
5556         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5557         if (!empty($secondValues)) {
5558           $secondOP = '<=';
5559           $secondPhrase = ts('greater than or equal to');
5560           $secondValue = $secondValues[2];
5561           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5562         }
5563       }
5564
5565       if ($secondOP) {
5566         $this->_where[$grouping][] = "
5567 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5568 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5569 ";
5570         $displayValue = $options ? $options[$value] : $value;
5571         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5572
5573         $this->_qill[$grouping][]
5574           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5575       }
5576       else {
5577         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5578         $displayValue = $options ? $options[$value] : $value;
5579         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5580       }
5581       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5582       return;
5583     }
5584   }
5585
5586   /**
5587    * Calculate date from age.
5588    *
5589    * @param string $asofDate
5590    * @param int $age
5591    * @param string $type
5592    *
5593    * @return string
5594    * @throws \Exception
5595    */
5596   public static function calcDateFromAge($asofDate, $age, $type) {
5597     $date = new DateTime($asofDate);
5598     if ($type == "min") {
5599       // minimum age is $age: dob <= date - age "235959"
5600       $date->sub(new DateInterval("P" . $age . "Y"));
5601       return $date->format('Ymd') . "235959";
5602     }
5603     else {
5604       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5605       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5606       return $date->format('Ymd') . "000000";
5607     }
5608   }
5609
5610   /**
5611    * Given the field name, operator, value & its data type
5612    * builds the where Clause for the query
5613    * used for handling 'IS NULL'/'IS NOT NULL' operators
5614    *
5615    * @param string $field
5616    *   Fieldname.
5617    * @param string $op
5618    *   Operator.
5619    * @param string $value
5620    *   Value.
5621    * @param string $dataType
5622    *   Data type of the field.
5623    *
5624    * @return string
5625    *   Where clause for the query.
5626    * @throws \CRM_Core_Exception
5627    */
5628   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5629     $op = trim($op);
5630     $clause = "$field $op";
5631
5632     switch ($op) {
5633       case 'IS NULL':
5634       case 'IS NOT NULL':
5635         return $clause;
5636
5637       case 'IS EMPTY':
5638         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5639         return $clause;
5640
5641       case 'IS NOT EMPTY':
5642         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5643         return $clause;
5644
5645       case 'RLIKE':
5646         return " {$clause} BINARY '{$value}' ";
5647
5648       case 'IN':
5649       case 'NOT IN':
5650         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5651         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5652           $value = [$op => (array) $value];
5653         }
5654
5655       default:
5656         if (empty($dataType) || $dataType == 'Date') {
5657           $dataType = 'String';
5658         }
5659         if (is_array($value)) {
5660           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5661           // widely used and consistent across the codebase
5662           // adding this here won't accept the search functions which don't submit an array
5663           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5664
5665             return $queryString;
5666           }
5667           if (!empty($value[0]) && $op === 'BETWEEN') {
5668             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5669             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5670               return $queryString;
5671             }
5672           }
5673           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5674         }
5675
5676         $value = CRM_Utils_Type::escape($value, $dataType);
5677         // if we don't have a dataType we should assume
5678         if ($dataType == 'String' || $dataType == 'Text') {
5679           $value = "'" . $value . "'";
5680         }
5681         return "$clause $value";
5682     }
5683   }
5684
5685   /**
5686    * @param bool $reset
5687    *
5688    * @return array
5689    */
5690   public function openedSearchPanes($reset = FALSE) {
5691     if (!$reset || empty($this->_whereTables)) {
5692       return self::$_openedPanes;
5693     }
5694
5695     // pane name to table mapper
5696     $panesMapper = [
5697       ts('Contributions') => 'civicrm_contribution',
5698       ts('Memberships') => 'civicrm_membership',
5699       ts('Events') => 'civicrm_participant',
5700       ts('Relationships') => 'civicrm_relationship',
5701       ts('Activities') => 'civicrm_activity',
5702       ts('Pledges') => 'civicrm_pledge',
5703       ts('Cases') => 'civicrm_case',
5704       ts('Grants') => 'civicrm_grant',
5705       ts('Address Fields') => 'civicrm_address',
5706       ts('Notes') => 'civicrm_note',
5707       ts('Change Log') => 'civicrm_log',
5708       ts('Mailings') => 'civicrm_mailing',
5709     ];
5710     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5711
5712     foreach (array_keys($this->_whereTables) as $table) {
5713       if ($panName = array_search($table, $panesMapper)) {
5714         self::$_openedPanes[$panName] = TRUE;
5715       }
5716     }
5717
5718     return self::$_openedPanes;
5719   }
5720
5721   /**
5722    * @param $operator
5723    */
5724   public function setOperator($operator) {
5725     $validOperators = ['AND', 'OR'];
5726     if (!in_array($operator, $validOperators)) {
5727       $operator = 'AND';
5728     }
5729     $this->_operator = $operator;
5730   }
5731
5732   /**
5733    * @return string
5734    */
5735   public function getOperator() {
5736     return $this->_operator;
5737   }
5738
5739   /**
5740    * @param $from
5741    * @param $where
5742    * @param $having
5743    */
5744   public function filterRelatedContacts(&$from, &$where, &$having) {
5745     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5746       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5747     }
5748     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5749     // since there only can be one instance of this filter in every query
5750     // skip if filter has already applied
5751     foreach ($_rTempCache as $acache) {
5752       foreach ($acache['queries'] as $aqcache) {
5753         if (strpos($from, $aqcache['from']) !== FALSE) {
5754           $having = NULL;
5755           return;
5756         }
5757       }
5758     }
5759     $arg_sig = sha1("$from $where $having");
5760     if (isset($_rTempCache[$arg_sig])) {
5761       $cache = $_rTempCache[$arg_sig];
5762     }
5763     else {
5764       // create temp table with contact ids
5765
5766       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5767
5768       $sql = "
5769 REPLACE INTO $tableName ( contact_id )
5770 SELECT contact_a.id
5771        $from
5772        $where
5773        $having
5774 ";
5775       CRM_Core_DAO::executeQuery($sql);
5776
5777       $cache = ['tableName' => $tableName, 'queries' => []];
5778       $_rTempCache[$arg_sig] = $cache;
5779     }
5780     // upsert the query depending on relationship type
5781     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5782       $qcache = $cache['queries'][$this->_displayRelationshipType];
5783     }
5784     else {
5785       $tableName = $cache['tableName'];
5786       $qcache = [
5787         "from" => "",
5788         "where" => "",
5789       ];
5790       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5791       if (is_numeric($this->_displayRelationshipType)) {
5792         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5793         $qcache['from'] = "
5794 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5795 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5796 ";
5797         $qcache['where'] = "
5798 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5799 AND   displayRelType.is_active = 1
5800 ";
5801       }
5802       else {
5803         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5804         if ($dirOne == 'a') {
5805           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5806           $qcache['from'] .= "
5807 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5808 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5809 ";
5810         }
5811         else {
5812           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5813           $qcache['from'] .= "
5814 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5815 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5816 ";
5817         }
5818         $qcache['where'] = "
5819 WHERE displayRelType.relationship_type_id = $relType
5820 AND   displayRelType.is_active = 1
5821 ";
5822       }
5823       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5824       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5825     }
5826     $qillMessage = ts('Contacts with a Relationship Type of: ');
5827     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5828     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5829       $this->_qill[0][] = $iqill;
5830     }
5831     if (strpos($from, $qcache['from']) === FALSE) {
5832       // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5833       // this happens when we have an event_type in the quert (CRM-7969)
5834       $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5835       $from .= $qcache['from'];
5836       $where = $qcache['where'];
5837       if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5838         $from .= " $this->_permissionFromClause";
5839       }
5840       if (!empty($this->_permissionWhereClause)) {
5841         $where .= "AND $this->_permissionWhereClause";
5842       }
5843     }
5844
5845     $having = NULL;
5846   }
5847
5848   /**
5849    * See CRM-19811 for why this is database hurty without apparent benefit.
5850    *
5851    * @param $op
5852    *
5853    * @return bool
5854    */
5855   public static function caseImportant($op) {
5856     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5857   }
5858
5859   /**
5860    * @param $returnProperties
5861    * @param $prefix
5862    *
5863    * @return bool
5864    */
5865   public static function componentPresent(&$returnProperties, $prefix) {
5866     foreach ($returnProperties as $name => $dontCare) {
5867       if (substr($name, 0, strlen($prefix)) == $prefix) {
5868         return TRUE;
5869       }
5870     }
5871     return FALSE;
5872   }
5873
5874   /**
5875    * Builds the necessary structures for all fields that are similar to option value look-ups.
5876    *
5877    * @param string $name
5878    *   the name of the field.
5879    * @param string $op
5880    *   the sql operator, this function should handle ALL SQL operators.
5881    * @param string $value
5882    *   depends on the operator and who's calling the query builder.
5883    * @param int $grouping
5884    *   the index where to place the where clause.
5885    * @param string $daoName
5886    *   DAO Name.
5887    * @param array $field
5888    *   an array that contains various properties of the field identified by $name.
5889    * @param string $label
5890    *   The label for this field element.
5891    * @param string $dataType
5892    *
5893    * @throws \CRM_Core_Exception
5894    */
5895   public function optionValueQuery(
5896     $name,
5897     $op,
5898     $value,
5899     $grouping,
5900     $daoName = NULL,
5901     $field,
5902     $label,
5903     $dataType = 'String'
5904   ) {
5905
5906     $pseudoFields = [
5907       'email_greeting',
5908       'postal_greeting',
5909       'addressee',
5910     ];
5911
5912     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5913     if ($tableName == 'civicrm_contact') {
5914       $wc = "contact_a.$fieldName";
5915     }
5916     else {
5917       // Special handling for on_hold, so that we actually use the 'where'
5918       // property in order to limit the query by the on_hold status of the email,
5919       // instead of using email.id which would be nonsensical.
5920       if ($field['name'] === 'on_hold') {
5921         $wc = $field['where'];
5922       }
5923       else {
5924         $wc = "$tableName.id";
5925       }
5926     }
5927
5928     if (in_array($name, $pseudoFields)) {
5929       $wc = "contact_a.{$name}_id";
5930       $dataType = 'Positive';
5931       $value = (!$value) ? 0 : $value;
5932     }
5933     if ($name == "world_region") {
5934       $field['name'] = $name;
5935     }
5936
5937     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5938     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5939     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5940   }
5941
5942   /**
5943    * Check and explode a user defined numeric string into an array
5944    * this was the protocol used by search builder in the old old days before we had
5945    * super nice js widgets to do the hard work
5946    *
5947    * @param string $string
5948    * @param string $dataType
5949    *   The dataType we should check for the values, default integer.
5950    *
5951    * @return bool|array
5952    *   false if string does not match the pattern
5953    *   array of numeric values if string does match the pattern
5954    */
5955   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5956     $string = trim($string);
5957     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5958       return FALSE;
5959     }
5960
5961     $string = substr($string, 1, -1);
5962     $values = explode(',', $string);
5963     if (empty($values)) {
5964       return FALSE;
5965     }
5966
5967     $returnValues = [];
5968     foreach ($values as $v) {
5969       if ($dataType == 'Integer' && !is_numeric($v)) {
5970         return FALSE;
5971       }
5972       elseif ($dataType == 'String' && !is_string($v)) {
5973         return FALSE;
5974       }
5975       $returnValues[] = trim($v);
5976     }
5977
5978     if (empty($returnValues)) {
5979       return FALSE;
5980     }
5981
5982     return $returnValues;
5983   }
5984
5985   /**
5986    * Convert the pseudo constants id's to their names
5987    *
5988    * @param CRM_Core_DAO $dao
5989    * @param bool $return
5990    * @param bool $usedForAPI
5991    *
5992    * @return array|NULL
5993    */
5994   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
5995     if (empty($this->_pseudoConstantsSelect)) {
5996       return NULL;
5997     }
5998     $values = [];
5999     foreach ($this->_pseudoConstantsSelect as $key => $value) {
6000       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
6001         continue;
6002       }
6003
6004       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6005         $val = $dao->{$value['idCol']};
6006         if ($key == 'groups') {
6007           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6008           continue;
6009         }
6010
6011         if (CRM_Utils_System::isNull($val)) {
6012           $dao->$key = NULL;
6013         }
6014         elseif (!empty($value['pseudoconstant'])) {
6015           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6016           // and we can use the metadata to figure it out.
6017           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6018           // more metadata based choices.
6019           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6020             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6021             foreach ($dbValues as $pseudoValue) {
6022               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6023             }
6024
6025             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6026             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
6027             if ($usedForAPI && $realFieldName) {
6028               // normally we would see 2 fields returned for pseudoConstants. An exception is
6029               // preferred_communication_method where there is no id-variant.
6030               // For the api we prioritise getting the real data returned.
6031               // over the resolved version
6032               $dao->$realFieldName = $dbValues;
6033             }
6034
6035           }
6036           else {
6037             // This is basically the same as the default but since we have the bao we can use
6038             // a cached function.
6039             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6040           }
6041         }
6042         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6043           //preserve id value
6044           $idColumn = "{$key}_id";
6045           $dao->$idColumn = $val;
6046
6047           if ($key == 'state_province_name') {
6048             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6049           }
6050           else {
6051             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6052           }
6053         }
6054         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6055           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6056         }
6057         // @todo handle this in the section above for pseudoconstants.
6058         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6059           // @todo define bao on this & merge into the above condition.
6060           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6061
6062           if ($value['pseudoField'] == 'participant_role') {
6063             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6064             foreach ($viewValues as $k => $v) {
6065               $viewValues[$k] = $pseudoOptions[$v];
6066             }
6067           }
6068           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6069         }
6070         else {
6071           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6072           $dao->$key = $labels[$val];
6073         }
6074
6075         // return converted values in array format
6076         if ($return) {
6077           if (strpos($key, '-') !== FALSE) {
6078             $keyVal = explode('-', $key);
6079             $current = &$values;
6080             $lastElement = array_pop($keyVal);
6081             foreach ($keyVal as $v) {
6082               if (!array_key_exists($v, $current)) {
6083                 $current[$v] = [];
6084               }
6085               $current = &$current[$v];
6086             }
6087             $current[$lastElement] = $dao->$key;
6088           }
6089           else {
6090             $values[$key] = $dao->$key;
6091           }
6092         }
6093       }
6094     }
6095     if (!$usedForAPI) {
6096       foreach ($this->legacyHackedFields as $realField => $labelField) {
6097         // This is a temporary routine for handling these fields while
6098         // we figure out how to handled them based on metadata in
6099         /// export and search builder. CRM-19815, CRM-19830.
6100         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6101           $dao->$realField = $dao->$labelField;
6102         }
6103       }
6104     }
6105     return $values;
6106   }
6107
6108   /**
6109    * Include pseudo fields LEFT JOIN.
6110    * @param string|array $sort can be a object or string
6111    *
6112    * @return array|NULL
6113    */
6114   public function includePseudoFieldsJoin($sort) {
6115     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6116       return NULL;
6117     }
6118     $sort = is_string($sort) ? $sort : $sort->orderBy();
6119     $present = [];
6120
6121     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6122       if (!empty($value['table'])) {
6123         $regex = "/({$value['table']}\.|{$name})/";
6124         if (preg_match($regex, $sort)) {
6125           $this->_elemnt[$value['element']] = 1;
6126           $this->_select[$value['element']] = $value['select'];
6127           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6128           $present[$value['table']] = $value['join'];
6129         }
6130       }
6131     }
6132     $presentSimpleFrom = $present;
6133
6134     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6135       array_key_exists('civicrm_country', $presentSimpleFrom)
6136     ) {
6137       unset($presentSimpleFrom['civicrm_country']);
6138     }
6139     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6140       array_key_exists('civicrm_country', $present)
6141     ) {
6142       unset($present['civicrm_country']);
6143     }
6144
6145     $presentClause = $presentSimpleFromClause = NULL;
6146     if (!empty($present)) {
6147       $presentClause = implode(' ', $present);
6148     }
6149     if (!empty($presentSimpleFrom)) {
6150       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6151     }
6152
6153     $this->_fromClause = $this->_fromClause . $presentClause;
6154     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6155
6156     return [$presentClause, $presentSimpleFromClause];
6157   }
6158
6159   /**
6160    * Build qill for field.
6161    *
6162    * Qill refers to the query detail visible on the UI.
6163    *
6164    * @param string $daoName
6165    * @param string $fieldName
6166    * @param mixed $fieldValue
6167    * @param string $op
6168    * @param array $pseudoExtraParam
6169    * @param int $type
6170    *   Type of the field per CRM_Utils_Type
6171    *
6172    * @return array
6173    */
6174   public static function buildQillForFieldValue(
6175     $daoName,
6176     $fieldName,
6177     $fieldValue,
6178     $op,
6179     $pseudoExtraParam = [],
6180     $type = CRM_Utils_Type::T_STRING
6181   ) {
6182     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6183
6184     //API usually have fieldValue format as array(operator => array(values)),
6185     //so we need to separate operator out of fieldValue param
6186     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6187       $op = key($fieldValue);
6188       $fieldValue = $fieldValue[$op];
6189     }
6190
6191     // if Operator chosen is NULL/EMPTY then
6192     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6193       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6194     }
6195
6196     // @todo - if the right BAO is passed in special handling for the below
6197     // fields should not be required. testQillOptions.
6198     if ($fieldName == 'country_id') {
6199       $pseudoOptions = CRM_Core_PseudoConstant::country();
6200     }
6201     elseif ($fieldName == 'county_id') {
6202       $pseudoOptions = CRM_Core_PseudoConstant::county();
6203     }
6204     elseif ($fieldName == 'world_region') {
6205       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6206     }
6207     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6208       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6209       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6210     }
6211     elseif ($fieldName == 'contribution_product_id') {
6212       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6213     }
6214     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6215       $pseudoOptions = CRM_Core_PseudoConstant::group();
6216     }
6217     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6218       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6219     }
6220     elseif ($daoName) {
6221       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6222     }
6223
6224     if (is_array($fieldValue)) {
6225       $qillString = [];
6226       if (!empty($pseudoOptions)) {
6227         foreach ((array) $fieldValue as $val) {
6228           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6229         }
6230         $fieldValue = implode(', ', $qillString);
6231       }
6232       else {
6233         if ($type == CRM_Utils_Type::T_DATE) {
6234           foreach ($fieldValue as $index => $value) {
6235             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6236           }
6237         }
6238         $separator = ', ';
6239         // @todo - this is a bit specific (one operator).
6240         // However it is covered by a unit test so can be altered later with
6241         // some confidence.
6242         if ($op === 'BETWEEN') {
6243           $separator = ' AND ';
6244         }
6245         $fieldValue = implode($separator, $fieldValue);
6246       }
6247     }
6248     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6249       $fieldValue = $pseudoOptions[$fieldValue];
6250     }
6251     elseif ($type === CRM_Utils_Type::T_DATE) {
6252       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6253     }
6254
6255     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6256   }
6257
6258   /**
6259    * Get the qill (search description for field) for the specified field.
6260    *
6261    * @param string $daoName
6262    * @param string $name
6263    * @param string $value
6264    * @param string|array $op
6265    * @param string $label
6266    *
6267    * @return string
6268    */
6269   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6270     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6271     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6272   }
6273
6274   /**
6275    * Alter value to reflect wildcard settings.
6276    *
6277    * The form will have tried to guess whether this is a good field to wildcard but there is
6278    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6279    * or only the end of the string
6280    *
6281    * @param bool $wildcard
6282    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6283    * @param string $op
6284    *   Generally '=' or 'LIKE'.
6285    * @param string $value
6286    *   The search string.
6287    *
6288    * @return string
6289    */
6290   public static function getWildCardedValue($wildcard, $op, $value) {
6291     if ($wildcard && $op === 'LIKE') {
6292       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6293         return "%$value%";
6294       }
6295       else {
6296         return "$value%";
6297       }
6298     }
6299     else {
6300       return "$value";
6301     }
6302   }
6303
6304   /**
6305    * Process special fields of Search Form in OK (Operator in Key) format
6306    *
6307    * @param array $formValues
6308    * @param array $specialFields
6309    *    Special params to be processed
6310    * @param array $changeNames
6311    *   Array of fields whose name should be changed
6312    */
6313   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6314     // Array of special fields whose value are considered only for NULL or EMPTY operators
6315     $nullableFields = ['contribution_batch_id'];
6316
6317     foreach ($specialFields as $element) {
6318       $value = $formValues[$element] ?? NULL;
6319       if ($value) {
6320         if (is_array($value)) {
6321           if (in_array($element, array_keys($changeNames))) {
6322             unset($formValues[$element]);
6323             $element = $changeNames[$element];
6324           }
6325           $formValues[$element] = ['IN' => $value];
6326         }
6327         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6328           $formValues[$element] = [$value => 1];
6329         }
6330         elseif (!in_array($element, $nullableFields)) {
6331           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6332           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6333           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6334         }
6335       }
6336     }
6337   }
6338
6339   /**
6340    * Parse and assimilate the various sort options.
6341    *
6342    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6343    * `email`), the related table may be joined automatically.
6344    *
6345    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6346    * it's been extracted to a standalone function.
6347    *
6348    * @param string|CRM_Utils_Sort $sort
6349    *   The order by string.
6350    * @param null $sortOrder
6351    *   Who knows? Hu knows. He who knows Hu knows who.
6352    *
6353    * @return string
6354    *   list(string $orderByClause, string $additionalFromClause).
6355    *
6356    * @throws \CRM_Core_Exception
6357    */
6358   protected function prepareOrderBy($sort, $sortOrder) {
6359     $orderByArray = [];
6360     $orderBy = '';
6361
6362     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6363       isset($this->_distinctComponentClause)
6364     ) {
6365       if ($sort) {
6366         if (is_string($sort)) {
6367           $orderBy = $sort;
6368         }
6369         else {
6370           $orderBy = trim($sort->orderBy());
6371         }
6372         // Deliberately remove the backticks again, as they mess up the evil
6373         // string munging below. This balanced by re-escaping before use.
6374         $orderBy = str_replace('`', '', $orderBy);
6375
6376         if (!empty($orderBy)) {
6377           // this is special case while searching for
6378           // change log CRM-1718
6379           if (preg_match('/sort_name/i', $orderBy)) {
6380             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6381           }
6382
6383           if ($sortOrder) {
6384             $orderBy .= " $sortOrder";
6385           }
6386
6387           // always add contact_a.id to the ORDER clause
6388           // so the order is deterministic
6389           if (strpos('contact_a.id', $orderBy) === FALSE) {
6390             $orderBy .= ", contact_a.id";
6391           }
6392         }
6393       }
6394       else {
6395         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6396       }
6397     }
6398     if (!$orderBy) {
6399       return NULL;
6400     }
6401     // Remove this here & add it at the end for simplicity.
6402     $order = trim($orderBy);
6403     $orderByArray = explode(',', $order);
6404
6405     foreach ($orderByArray as $orderByClause) {
6406       $orderByClauseParts = explode(' ', trim($orderByClause));
6407       $field = $orderByClauseParts[0];
6408       $direction = $orderByClauseParts[1] ?? 'asc';
6409       $fieldSpec = $this->getMetadataForRealField($field);
6410
6411       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6412       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6413       // add keys to $this->fields to represent the actual keys for locations.
6414       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6415         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6416         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6417       }
6418
6419       if ($this->_returnProperties === []) {
6420         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6421           $this->_tables[$fieldSpec['table_name']] = 1;
6422           $order = $fieldSpec['where'] . ' ' . $direction;
6423         }
6424
6425       }
6426       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6427       // add to cfIDs array if not present
6428       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6429         $this->_cfIDs[$cfID] = [];
6430         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6431         $this->_customQuery->query();
6432         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6433         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6434       }
6435
6436       // By replacing the join to the option value table with the mysql construct
6437       // ORDER BY field('contribution_status_id', 2,1,4)
6438       // we can remove a join. In the case of the option value join it is
6439       /// a join known to cause slow queries.
6440       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6441       // matching an id+name parrern in the
6442       // first instance for scope reasons. They require slightly different handling as the column (label)
6443       // is not declared for them.
6444       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6445       // escaping.
6446       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6447       if (!empty($pseudoConstantMetadata)
6448       ) {
6449         if (!empty($pseudoConstantMetadata['optionGroupName'])
6450           || $this->isPseudoFieldAnFK($fieldSpec)
6451         ) {
6452           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6453           if (empty($fieldSpec['bao'])) {
6454             continue;
6455           }
6456           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name']);
6457           natcasesort($sortedOptions);
6458           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6459           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6460           // this might give some early warning.
6461           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6462           // use where if it's set to fully qualify ambiguous column names
6463           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6464           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6465           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6466         }
6467         //CRM-12565 add "`" around $field if it is a pseudo constant
6468         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6469         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6470           $order = str_replace($field, "`{$field}`", $order);
6471         }
6472       }
6473     }
6474
6475     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6476     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6477
6478     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6479     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6480     if ($order) {
6481       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6482       return ' ORDER BY ' . $order;
6483     }
6484   }
6485
6486   /**
6487    * Convert a string of group IDs to a string of group labels.
6488    *
6489    * The original string may include duplicates and groups the user does not have
6490    * permission to see.
6491    *
6492    * @param CRM_Core_DAO $dao
6493    * @param string $val
6494    *
6495    * @return string
6496    */
6497   public function convertGroupIDStringToLabelString(&$dao, $val) {
6498     $groupIDs = explode(',', $val);
6499     // Note that groups that the user does not have permission to will be excluded (good).
6500     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6501     return implode(', ', $groups);
6502
6503   }
6504
6505   /**
6506    * Set the qill and where properties for a field.
6507    *
6508    * This function is intended as a short-term function to encourage refactoring
6509    * & re-use - but really we should just have less special-casing.
6510    *
6511    * @param string $name
6512    * @param string $op
6513    * @param string|array $value
6514    * @param string $grouping
6515    * @param array $field
6516    *
6517    * @throws \CRM_Core_Exception
6518    */
6519   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6520     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6521     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6522     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6523       1 => $field['title'],
6524       2 => $qillop,
6525       3 => $qillVal,
6526     ]);
6527   }
6528
6529   /**
6530    * Has the pseudoconstant of the field been requested.
6531    *
6532    * For example if the field is payment_instrument_id then it
6533    * has been requested if either payment_instrument_id or payment_instrument
6534    * have been requested. Payment_instrument is the option groun name field value.
6535    *
6536    * @param array $field
6537    * @param string $fieldName
6538    *   The unique name of the field - ie. the one it will be aliased to in the query.
6539    *
6540    * @return bool
6541    */
6542   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6543     $realField = $this->getMetadataForRealField($fieldName);
6544     if (!isset($realField['pseudoconstant'])) {
6545       return FALSE;
6546     }
6547     $pseudoConstant = $realField['pseudoconstant'];
6548     if (empty($pseudoConstant['optionGroupName']) &&
6549       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6550       // We are increasing our pseudoconstant handling - but still very cautiously,
6551       // hence the check for labelColumn === name
6552       return FALSE;
6553     }
6554
6555     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6556       return TRUE;
6557     }
6558     if (!empty($this->_returnProperties[$fieldName])) {
6559       return TRUE;
6560     }
6561     // Is this still required - the above goes off the unique name. Test with things like
6562     // communication_preferences & prefix_id.
6563     if (!empty($this->_returnProperties[$field['name']])) {
6564       return TRUE;
6565     }
6566     return FALSE;
6567   }
6568
6569   /**
6570    * Get Select Clause.
6571    *
6572    * @return string
6573    */
6574   public function getSelect() {
6575     $select = 'SELECT ';
6576     if (isset($this->_distinctComponentClause)) {
6577       $select .= "{$this->_distinctComponentClause}, ";
6578     }
6579     $select .= implode(', ', $this->_select);
6580     return $select;
6581   }
6582
6583   /**
6584    * Add basic statistics to the summary.
6585    *
6586    * @param array $summary
6587    * @param string $where
6588    * @param string $from
6589    *
6590    * @return array
6591    * @throws \CRM_Core_Exception
6592    */
6593   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6594     $summary['total']['count'] = 0;
6595     $summary['total']['amount'] = $summary['total']['avg'] = [];
6596
6597     $query = "
6598       SELECT COUNT( conts.total_amount ) as total_count,
6599         SUM(   conts.total_amount ) as total_amount,
6600         AVG(   conts.total_amount ) as total_avg,
6601         conts.currency              as currency
6602       FROM (
6603         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6604         civicrm_contribution.currency
6605         $from
6606         $where AND civicrm_contribution.contribution_status_id = 1
6607       GROUP BY civicrm_contribution.id
6608     ) as conts
6609     GROUP BY currency";
6610
6611     $dao = CRM_Core_DAO::executeQuery($query);
6612
6613     while ($dao->fetch()) {
6614       $summary['total']['count'] += $dao->total_count;
6615       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6616       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6617     }
6618
6619     if (!empty($summary['total']['amount'])) {
6620       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6621       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6622     }
6623     else {
6624       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6625     }
6626     return $summary;
6627   }
6628
6629   /**
6630    * Add basic soft credit statistics to summary array.
6631    *
6632    * @param array $summary
6633    * @param string $where
6634    * @param string $from
6635    *
6636    * @throws \CRM_Core_Exception
6637    */
6638   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6639     $query = "
6640       SELECT COUNT( conts.total_amount ) as total_count,
6641       SUM(   conts.total_amount ) as total_amount,
6642       AVG(   conts.total_amount ) as total_avg,
6643       conts.currency as currency
6644       FROM (
6645         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6646         $from
6647         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6648         GROUP BY civicrm_contribution_soft.id
6649       ) as conts
6650       GROUP BY currency";
6651
6652     $dao = CRM_Core_DAO::executeQuery($query);
6653     $summary['soft_credit']['count'] = 0;
6654     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6655     while ($dao->fetch()) {
6656       $summary['soft_credit']['count'] += $dao->total_count;
6657       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6658       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6659     }
6660     if (!empty($summary['soft_credit']['amount'])) {
6661       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6662       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6663     }
6664     else {
6665       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6666     }
6667   }
6668
6669   /**
6670    * Add basic stats about cancelled contributions to the summary.
6671    *
6672    * @param array $summary
6673    * @param string $where
6674    * @param string $from
6675    *
6676    * @throws \CRM_Core_Exception
6677    */
6678   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6679     $query = "
6680       SELECT COUNT( conts.total_amount ) as cancel_count,
6681        SUM(   conts.total_amount ) as cancel_amount,
6682        AVG(   conts.total_amount ) as cancel_avg,
6683        conts.currency              as currency
6684         FROM (
6685       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6686       $from
6687       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6688       GROUP BY civicrm_contribution.id
6689     ) as conts
6690     GROUP BY currency";
6691
6692     $dao = CRM_Core_DAO::executeQuery($query);
6693
6694     if ($dao->N <= 1) {
6695       if ($dao->fetch()) {
6696         $summary['cancel']['count'] = $dao->cancel_count;
6697         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6698         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6699       }
6700     }
6701     else {
6702       $summary['cancel']['count'] = 0;
6703       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6704       while ($dao->fetch()) {
6705         $summary['cancel']['count'] += $dao->cancel_count;
6706         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6707         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6708       }
6709       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6710       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6711     }
6712   }
6713
6714   /**
6715    * Create the sql query for an contact search.
6716    *
6717    * @param int $offset
6718    *   The offset for the query.
6719    * @param int $rowCount
6720    *   The number of rows to return.
6721    * @param string|CRM_Utils_Sort $sort
6722    *   The order by string.
6723    * @param bool $count
6724    *   Is this a count only query ?.
6725    * @param bool $includeContactIds
6726    *   Should we include contact ids?.
6727    * @param bool $sortByChar
6728    *   If true returns the distinct array of first characters for search results.
6729    * @param bool $groupContacts
6730    *   If true, return only the contact ids.
6731    * @param string $additionalWhereClause
6732    *   If the caller wants to further restrict the search (used for components).
6733    * @param null $sortOrder
6734    * @param string $additionalFromClause
6735    *   Should be clause with proper joins, effective to reduce where clause load.
6736    *
6737    * @param bool $skipOrderAndLimit
6738    *
6739    * @return string
6740    *
6741    * @throws \CRM_Core_Exception
6742    */
6743   public function getSearchSQL(
6744     $offset = 0, $rowCount = 0, $sort = NULL,
6745     $count = FALSE, $includeContactIds = FALSE,
6746     $sortByChar = FALSE, $groupContacts = FALSE,
6747     $additionalWhereClause = NULL, $sortOrder = NULL,
6748     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6749
6750     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6751
6752     if ($sortByChar) {
6753       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6754       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6755     }
6756
6757     if ($skipOrderAndLimit) {
6758       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6759       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6760     }
6761     else {
6762       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6763     }
6764     return $query;
6765   }
6766
6767   /**
6768    * Get the component parts of the search query as an array.
6769    *
6770    * @param int $offset
6771    *   The offset for the query.
6772    * @param int $rowCount
6773    *   The number of rows to return.
6774    * @param string|CRM_Utils_Sort $sort
6775    *   The order by string.
6776    * @param bool $count
6777    *   Is this a count only query ?.
6778    * @param bool $includeContactIds
6779    *   Should we include contact ids?.
6780    * @param bool $sortByChar
6781    *   If true returns the distinct array of first characters for search results.
6782    * @param bool $groupContacts
6783    *   If true, return only the contact ids.
6784    * @param string $additionalWhereClause
6785    *   If the caller wants to further restrict the search (used for components).
6786    * @param null $sortOrder
6787    * @param string $additionalFromClause
6788    *   Should be clause with proper joins, effective to reduce where clause load.
6789    *
6790    * @return array
6791    * @throws \CRM_Core_Exception
6792    */
6793   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6794     $count = FALSE, $includeContactIds = FALSE,
6795     $sortByChar = FALSE, $groupContacts = FALSE,
6796     $additionalWhereClause = NULL, $sortOrder = NULL,
6797     $additionalFromClause = NULL) {
6798     if ($includeContactIds) {
6799       $this->_includeContactIds = TRUE;
6800       $this->_whereClause = $this->whereClause();
6801     }
6802     $onlyDeleted = in_array([
6803       'deleted_contacts',
6804       '=',
6805       '1',
6806       '0',
6807       '0',
6808     ], $this->_params);
6809
6810     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6811     // and that contact happens to be deleted, set $onlyDeleted to true
6812     foreach ($this->_params as $values) {
6813       $name = $values[0] ?? NULL;
6814       $op = $values[1] ?? NULL;
6815       $value = $values[2] ?? NULL;
6816       if ($name === 'contact_id' and $op === '=') {
6817         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6818           $onlyDeleted = TRUE;
6819         }
6820         break;
6821       }
6822     }
6823
6824     // building the query string
6825     $groupBy = $groupByCols = NULL;
6826     if (!$count) {
6827       if (isset($this->_groupByComponentClause)) {
6828         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6829         $groupByCols = explode(', ', $groupByCols);
6830       }
6831       elseif ($this->_useGroupBy) {
6832         $groupByCols = ['contact_a.id'];
6833       }
6834     }
6835     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6836       $groupByCols = ['civicrm_activity.id'];
6837     }
6838     if (!empty($groupByCols)) {
6839       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6840     }
6841
6842     $order = $orderBy = '';
6843     if (!$count) {
6844       if (!$sortByChar) {
6845         $order = $this->prepareOrderBy($sort, $sortOrder);
6846       }
6847     }
6848     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6849     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6850     //     clause. This will impact the search query output.
6851     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6852
6853     if ($disableFullGroupByMode) {
6854       CRM_Core_DAO::disableFullGroupByMode();
6855     }
6856
6857     // CRM-15231
6858     $this->_sort = $sort;
6859
6860     //CRM-15967
6861     $this->includePseudoFieldsJoin($sort);
6862
6863     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6864
6865     if ($additionalWhereClause) {
6866       $where = $where . ' AND ' . $additionalWhereClause;
6867     }
6868
6869     //additional from clause should be w/ proper joins.
6870     if ($additionalFromClause) {
6871       $from .= "\n" . $additionalFromClause;
6872     }
6873
6874     // if we are doing a transform, do it here
6875     // use the $from, $where and $having to get the contact ID
6876     if ($this->_displayRelationshipType) {
6877       $this->filterRelatedContacts($from, $where, $having);
6878     }
6879     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6880
6881     return [
6882       'select' => $select,
6883       'from' => $from,
6884       'where' => $where,
6885       'order_by' => $order,
6886       'group_by' => $groupBy,
6887       'having' => $having,
6888       'limit' => $limit,
6889     ];
6890   }
6891
6892   /**
6893    * Get the metadata for a given field.
6894    *
6895    * @param string $fieldName
6896    *
6897    * @return array
6898    */
6899   protected function getMetadataForField($fieldName) {
6900     if ($fieldName === 'contact_a.id') {
6901       // This seems to be the only anomaly.
6902       $fieldName = 'id';
6903     }
6904     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6905     $field = $this->_fields[$fieldName] ?? $pseudoField;
6906     $field = array_merge($field, $pseudoField);
6907     if (!empty($field) && empty($field['name'])) {
6908       // standardising field formatting here - over time we can phase out variants.
6909       // all paths using this currently unit tested
6910       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6911     }
6912     return $field;
6913   }
6914
6915   /**
6916    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6917    *
6918    * @param string $fieldName
6919    *
6920    * @return array
6921    */
6922   public function getMetadataForRealField($fieldName) {
6923     $field = $this->getMetadataForField($fieldName);
6924     if (!empty($field['is_pseudofield_for'])) {
6925       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6926       $field['pseudofield_name'] = $fieldName;
6927     }
6928     elseif (!empty($field['pseudoconstant'])) {
6929       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6930         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6931         if (empty($field['table_name'])) {
6932           if (!empty($field['where'])) {
6933             $field['table_name'] = explode('.', $field['where'])[0];
6934           }
6935           else {
6936             $field['table_name'] = 'civicrm_contact';
6937           }
6938         }
6939       }
6940     }
6941     return $field;
6942   }
6943
6944   /**
6945    * Get the field datatype, using the type in the database rather than the pseudofield, if a pseudofield.
6946    *
6947    * @param string $fieldName
6948    *
6949    * @return string
6950    */
6951   public function getDataTypeForRealField($fieldName) {
6952     return CRM_Utils_Type::typeToString($this->getMetadataForRealField($fieldName)['type']);
6953   }
6954
6955   /**
6956    * If we have a field that is better rendered via the pseudoconstant handled them here.
6957    *
6958    * Rather than joining in the additional table we render the option value on output.
6959    *
6960    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6961    * carefully with test coverage but aim to extend.
6962    *
6963    * @param string $name
6964    */
6965   protected function addPseudoconstantFieldToSelect($name) {
6966     $field = $this->getMetadataForRealField($name);
6967     $realFieldName = $field['name'];
6968     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6969     if ($pseudoFieldName) {
6970       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6971       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6972         'pseudoField' => $pseudoFieldName,
6973         'idCol' => $realFieldName,
6974         'field_name' => $field['name'],
6975         'bao' => $field['bao'],
6976         'pseudoconstant' => $field['pseudoconstant'],
6977       ];
6978     }
6979
6980     $this->_tables[$field['table_name']] = 1;
6981     $this->_element[$realFieldName] = 1;
6982     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6983   }
6984
6985   /**
6986    * Is this pseudofield a foreign key constraint.
6987    *
6988    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
6989    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
6990    * array which is done for contributions which are mostly handled as pseudoconstants.
6991    *
6992    * @param $fieldSpec
6993    *
6994    * @return bool
6995    */
6996   protected function isPseudoFieldAnFK($fieldSpec) {
6997     if (empty($fieldSpec['FKClassName'])
6998       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
6999       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
7000       return FALSE;
7001     }
7002     return TRUE;
7003   }
7004
7005   /**
7006    * Is the field a relative date field.
7007    *
7008    * @param string $fieldName
7009    *
7010    * @return bool
7011    */
7012   protected function isARelativeDateField($fieldName) {
7013     if (substr($fieldName, -9, 9) !== '_relative') {
7014       return FALSE;
7015     }
7016     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
7017     return isset($this->_fields[$realField]);
7018   }
7019
7020   /**
7021    * Get the specifications for the field, if available.
7022    *
7023    * @param string $fieldName
7024    *   Fieldname as displayed on the form.
7025    *
7026    * @return array
7027    */
7028   public function getFieldSpec($fieldName) {
7029     if (isset($this->_fields[$fieldName])) {
7030       $fieldSpec = $this->_fields[$fieldName];
7031       if (!empty($fieldSpec['is_pseudofield_for'])) {
7032         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
7033       }
7034       return $fieldSpec;
7035     }
7036     $lowFieldName = str_replace('_low', '', $fieldName);
7037     if (isset($this->_fields[$lowFieldName])) {
7038       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
7039     }
7040     $highFieldName = str_replace('_high', '', $fieldName);
7041     if (isset($this->_fields[$highFieldName])) {
7042       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
7043     }
7044     return [];
7045   }
7046
7047   public function buildWhereForDate() {
7048
7049   }
7050
7051   /**
7052    * Is the field a relative date field.
7053    *
7054    * @param string $fieldName
7055    *
7056    * @return bool
7057    */
7058   protected function isADateRangeField($fieldName) {
7059     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7060       return FALSE;
7061     }
7062     return !empty($this->getFieldSpec($fieldName));
7063   }
7064
7065   /**
7066    * @param $values
7067    */
7068   protected function buildRelativeDateQuery(&$values) {
7069     $value = $values[2] ?? NULL;
7070     if (empty($value)) {
7071       return;
7072     }
7073     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7074     $fieldSpec = $this->_fields[$fieldName];
7075     $tableName = $fieldSpec['table_name'];
7076     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7077     $grouping = $values[3] ?? NULL;
7078     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7079     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7080     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7081
7082     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7083     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7084     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7085
7086     $where = $fieldSpec['where'];
7087     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7088       // Special handling for contact table as it has a known alias in advanced search.
7089       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7090       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7091     }
7092
7093     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7094     if ($fieldName === 'relation_active_period_date') {
7095       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7096       // Not sure the  'right' fix.
7097       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7098       return;
7099     }
7100
7101     if (empty($dates[0])) {
7102       // ie. no start date we only have end date
7103       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7104     }
7105     elseif (empty($dates[1])) {
7106
7107       // ie. no end date we only have start date
7108       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7109     }
7110     else {
7111       // we have start and end dates.
7112       if ($secondWhere !== $where) {
7113         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7114       }
7115       else {
7116         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7117       }
7118     }
7119   }
7120
7121   /**
7122    * Build the query for a date field if it is a _high or _low field.
7123    *
7124    * @param $values
7125    *
7126    * @return bool
7127    * @throws \CRM_Core_Exception
7128    */
7129   public function buildDateRangeQuery($values) {
7130     if ($this->isADateRangeField($values[0])) {
7131       $fieldSpec = $this->getFieldSpec($values[0]);
7132       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7133       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7134       return TRUE;
7135     }
7136     return FALSE;
7137   }
7138
7139   /**
7140    * Add the address table into the query.
7141    *
7142    * @param string $tableKey
7143    * @param string $joinCondition
7144    *
7145    * @return array
7146    *   - alias name
7147    *   - address join.
7148    */
7149   protected function addAddressTable($tableKey, $joinCondition) {
7150     $tName = "$tableKey-address";
7151     $aName = "`$tableKey-address`";
7152     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7153     $this->_element["{$tName}_id"] = 1;
7154     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7155     $this->_tables[$tName] = $addressJoin;
7156
7157     return [
7158       $aName,
7159       $addressJoin,
7160     ];
7161   }
7162
7163   /**
7164    * Get the clause for group status.
7165    *
7166    * @param int $grouping
7167    *
7168    * @return string
7169    */
7170   protected function getGroupStatusClause($grouping) {
7171     $statuses = $this->getSelectedGroupStatuses($grouping);
7172     return "status IN (" . implode(', ', $statuses) . ")";
7173   }
7174
7175   /**
7176    * Get an array of the statuses that have been selected.
7177    *
7178    * @param string $grouping
7179    *
7180    * @return array
7181    *
7182    * @throws \CRM_Core_Exception
7183    */
7184   protected function getSelectedGroupStatuses($grouping) {
7185     $statuses = [];
7186     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7187     if ($gcsValues &&
7188       is_array($gcsValues[2])
7189     ) {
7190       foreach ($gcsValues[2] as $k => $v) {
7191         if ($v) {
7192           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7193         }
7194       }
7195     }
7196     else {
7197       $statuses[] = "'Added'";
7198     }
7199     return $statuses;
7200   }
7201
7202   /**
7203    * Get the qill value for the field.
7204    *
7205    * @param string $name
7206    * @param array|int|string $value
7207    * @param string $op
7208    * @param array $fieldSpec
7209    * @param string $labelOverride
7210    *   Label override, if required.
7211    *
7212    * @return string
7213    */
7214   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7215     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7216     return (string) ts("%1 %2 %3", [
7217       1 => $labelOverride ?? $fieldSpec['title'],
7218       2 => $qillop,
7219       3 => $qillVal,
7220     ]);
7221   }
7222
7223   /**
7224    * Where handling for any field with adequately defined metadata.
7225    *
7226    * @param array $fieldSpec
7227    * @param string $name
7228    * @param string|array|int $value
7229    * @param string $op
7230    * @param string|int $grouping
7231    *
7232    * @throws \CRM_Core_Exception
7233    */
7234   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7235     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7236     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7237     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7238       $this->_tables[$fieldSpec['table_name']] = 1;
7239     }
7240     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7241       $this->_whereTables[$fieldSpec['table_name']] = 1;
7242     }
7243   }
7244
7245   /**
7246    * Get the qill for the relative date range.
7247    *
7248    * @param string|null $from
7249    * @param string|null $to
7250    * @param string $fieldTitle
7251    * @param string $relativeRange
7252    *
7253    * @return string
7254    */
7255   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7256     if (!$from) {
7257       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7258     }
7259     if (!$to) {
7260       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7261     }
7262     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7263       CRM_Utils_Date::customFormat($from),
7264       CRM_Utils_Date::customFormat($to),
7265     ]) . ')';
7266   }
7267
7268 }