CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * The cache to translate the option values into labels.
 218    *
 219    * @var array
 220    */
 221   public $_options;
 222
 223   /**
 224    * Are we in search mode.
 225    *
 226    * @var bool
 227    */
 228   public $_search = TRUE;
 229
 230   /**
 231    * Should we skip permission checking.
 232    *
 233    * @var bool
 234    */
 235   public $_skipPermission = FALSE;
 236
 237   /**
 238    * Should we skip adding of delete clause.
 239    *
 240    * @var bool
 241    */
 242   public $_skipDeleteClause = FALSE;
 243
 244   /**
 245    * Are we in strict mode (use equality over LIKE)
 246    *
 247    * @var bool
 248    */
 249   public $_strict = FALSE;
 250
 251   /**
 252    * What operator to use to group the clauses.
 253    *
 254    * @var string
 255    */
 256   public $_operator = 'AND';
 257
 258   public $_mode = 1;
 259
 260   /**
 261    * Should we only search on primary location.
 262    *
 263    * @var bool
 264    */
 265   public $_primaryLocation = TRUE;
 266
 267   /**
 268    * Are contact ids part of the query.
 269    *
 270    * @var bool
 271    */
 272   public $_includeContactIds = FALSE;
 273
 274   /**
 275    * Should we use the smart group cache.
 276    *
 277    * @var bool
 278    */
 279   public $_smartGroupCache = TRUE;
 280
 281   /**
 282    * Should we display contacts with a specific relationship type.
 283    *
 284    * @var string
 285    */
 286   public $_displayRelationshipType;
 287
 288   /**
 289    * Reference to the query object for custom values.
 290    *
 291    * @var Object
 292    */
 293   public $_customQuery;
 294
 295   /**
 296    * Should we enable the distinct clause, used if we are including
 297    * more than one group
 298    *
 299    * @var bool
 300    */
 301   public $_useDistinct = FALSE;
 302
 303   /**
 304    * Should we just display one contact record
 305    * @var bool
 306    */
 307   public $_useGroupBy = FALSE;
 308
 309   /**
 310    * The relationship type direction
 311    *
 312    * @var array
 313    */
 314   public static $_relType;
 315
 316   /**
 317    * The activity role
 318    *
 319    * @var array
 320    */
 321   public static $_activityRole;
 322
 323   /**
 324    * Consider the component activity type
 325    * during activity search.
 326    *
 327    * @var array
 328    */
 329   public static $_considerCompActivities;
 330
 331   /**
 332    * Consider with contact activities only,
 333    * during activity search.
 334    *
 335    * @var array
 336    */
 337   public static $_withContactActivitiesOnly;
 338
 339   /**
 340    * Use distinct component clause for component searches
 341    *
 342    * @var string
 343    */
 344   public $_distinctComponentClause;
 345
 346   public $_rowCountClause;
 347
 348   /**
 349    * Use groupBy component clause for component searches
 350    *
 351    * @var string
 352    */
 353   public $_groupByComponentClause;
 354
 355   /**
 356    * Track open panes, useful in advance search
 357    *
 358    * @var array
 359    */
 360   public static $_openedPanes = [];
 361
 362   /**
 363    * For search builder - which custom fields are location-dependent
 364    * @var array
 365    */
 366   public $_locationSpecificCustomFields = [];
 367
 368   /**
 369    * The tables which have a dependency on location and/or address
 370    *
 371    * @var array
 372    */
 373   public static $_dependencies = [
 374     'civicrm_state_province' => 1,
 375     'civicrm_country' => 1,
 376     'civicrm_county' => 1,
 377     'civicrm_address' => 1,
 378     'civicrm_location_type' => 1,
 379   ];
 380
 381   /**
 382    * List of location specific fields.
 383    * @var array
 384    */
 385   public static $_locationSpecificFields = [
 386     'street_address',
 387     'street_number',
 388     'street_name',
 389     'street_unit',
 390     'supplemental_address_1',
 391     'supplemental_address_2',
 392     'supplemental_address_3',
 393     'city',
 394     'postal_code',
 395     'postal_code_suffix',
 396     'geo_code_1',
 397     'geo_code_2',
 398     'state_province',
 399     'country',
 400     'county',
 401     'phone',
 402     'email',
 403     'im',
 404     'address_name',
 405     'master_id',
 406   ];
 407
 408   /**
 409    * Remember if we handle either end of a number or date range
 410    * so we can skip the other
 411    * @var array
 412    */
 413   protected $_rangeCache = [];
 414   /**
 415    * Set to true when $this->relationship is run to avoid adding twice.
 416    *
 417    * @var bool
 418    */
 419   protected $_relationshipValuesAdded = FALSE;
 420
 421   /**
 422    * Set to the name of the temp table if one has been created.
 423    *
 424    * @var string
 425    */
 426   public static $_relationshipTempTable;
 427
 428   public $_pseudoConstantsSelect = [];
 429
 430   public $_groupUniqueKey;
 431   public $_groupKeys = [];
 432
 433   /**
 434    * Class constructor which also does all the work.
 435    *
 436    * @param array $params
 437    * @param array $returnProperties
 438    * @param array $fields
 439    * @param bool $includeContactIds
 440    * @param bool $strict
 441    * @param bool|int $mode - mode the search is operating on
 442    *
 443    * @param bool $skipPermission
 444    * @param bool $searchDescendentGroups
 445    * @param bool $smartGroupCache
 446    * @param null $displayRelationshipType
 447    * @param string $operator
 448    * @param string $apiEntity
 449    * @param bool|null $primaryLocationOnly
 450    *
 451    * @throws \CRM_Core_Exception
 452    */
 453   public function __construct(
 454     $params = NULL, $returnProperties = NULL, $fields = NULL,
 455     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 456     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 457     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 458     $operator = 'AND',
 459     $apiEntity = NULL,
 460     $primaryLocationOnly = NULL
 461   ) {
 462     if ($primaryLocationOnly === NULL) {
 463       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 464     }
 465     $this->_primaryLocation = $primaryLocationOnly;
 466     $this->_params = &$params;
 467     if ($this->_params == NULL) {
 468       $this->_params = [];
 469     }
 470
 471     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 472       $this->_returnProperties = [];
 473     }
 474     elseif (empty($returnProperties)) {
 475       $this->_returnProperties = self::defaultReturnProperties($mode);
 476     }
 477     else {
 478       $this->_returnProperties = &$returnProperties;
 479     }
 480
 481     $this->_includeContactIds = $includeContactIds;
 482     $this->_strict = $strict;
 483     $this->_mode = $mode;
 484     $this->_skipPermission = $skipPermission;
 485     $this->_smartGroupCache = $smartGroupCache;
 486     $this->_displayRelationshipType = $displayRelationshipType;
 487     $this->setOperator($operator);
 488
 489     if ($fields) {
 490       $this->_fields = &$fields;
 491       $this->_search = FALSE;
 492       $this->_skipPermission = TRUE;
 493     }
 494     else {
 495       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 496       //  The legacy hacked fields will output as a string rather than their underlying type.
 497       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 498         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 499       }
 500       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 501       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 502       // No good option other than hard-coding metadata for this 'special' field in.
 503       $relationFields['relation_active_period_date'] = [
 504         'name' => 'relation_active_period_date',
 505         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 506         'title' => ts('Active Period'),
 507         'table_name' => 'civicrm_relationship',
 508         'where' => 'civicrm_relationship.start_date',
 509         'where_end' => 'civicrm_relationship.end_date',
 510         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 511       ];
 512       $this->_fields = array_merge($relationFields, $this->_fields);
 513
 514       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 515       unset($fields['note']);
 516       $this->_fields = array_merge($this->_fields, $fields);
 517
 518       // add activity fields
 519       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 520       // Add hack as no unique name is defined for the field but the search form is in denial.
 521       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 522
 523       // add any fields provided by hook implementers
 524       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 525       $this->_fields = array_merge($this->_fields, $extFields);
 526     }
 527
 528     // basically do all the work once, and then reuse it
 529     $this->initialize($apiEntity);
 530   }
 531
 532   /**
 533    * Function which actually does all the work for the constructor.
 534    *
 535    * @param string $apiEntity
 536    *   The api entity being called.
 537    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 538    *
 539    * @throws \CRM_Core_Exception
 540    */
 541   public function initialize($apiEntity = NULL) {
 542     $this->_select = [];
 543     $this->_element = [];
 544     $this->_tables = [];
 545     $this->_whereTables = [];
 546     $this->_where = [];
 547     $this->_qill = [];
 548     $this->_options = [];
 549     $this->_cfIDs = [];
 550     $this->_paramLookup = [];
 551     $this->_having = [];
 552
 553     $this->_customQuery = NULL;
 554
 555     // reset cached static variables - CRM-5803
 556     self::$_activityRole = NULL;
 557     self::$_considerCompActivities = NULL;
 558     self::$_withContactActivitiesOnly = NULL;
 559
 560     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 561     $this->_element['contact_id'] = 1;
 562     $this->_tables['civicrm_contact'] = 1;
 563
 564     if (!empty($this->_params)) {
 565       $this->buildParamsLookup();
 566     }
 567
 568     $this->_whereTables = $this->_tables;
 569
 570     $this->selectClause($apiEntity);
 571     if (!empty($this->_cfIDs)) {
 572       // @todo This function is the select function but instead of running 'select' it
 573       // is running the whole query.
 574       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 575       $this->_customQuery->query();
 576       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 577       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 578       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 579       $this->_options = $this->_customQuery->_options;
 580     }
 581     $isForcePrimaryOnly = !empty($apiEntity);
 582     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 583     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 584       $component = 'contribution';
 585     }
 586     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 587       $component = 'membership';
 588     }
 589     if (isset($component) && !$this->_skipPermission) {
 590       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 591       CRM_Financial_BAO_FinancialType::buildPermissionedClause($this->_whereClause, $component);
 592     }
 593
 594     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 595     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 596
 597     $this->openedSearchPanes(TRUE);
 598   }
 599
 600   /**
 601    * Function for same purpose as convertFormValues.
 602    *
 603    * Like convert form values this function exists to pre-Process parameters from the form.
 604    *
 605    * It is unclear why they are different functions & likely relates to advances search
 606    * versus search builder.
 607    *
 608    * The direction we are going is having the form convert values to a standardised format &
 609    * moving away from weird & wonderful where clause switches.
 610    *
 611    * Fix and handle contact deletion nicely.
 612    *
 613    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 614    *
 615    * CRM-11971
 616    */
 617   public function buildParamsLookup() {
 618     $trashParamExists = FALSE;
 619     $paramByGroup = [];
 620     foreach ($this->_params as $k => $param) {
 621       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 622         $trashParamExists = TRUE;
 623       }
 624       if (!empty($param[3])) {
 625         $paramByGroup[$param[3]][$k] = $param;
 626       }
 627     }
 628
 629     if ($trashParamExists) {
 630       $this->_skipDeleteClause = TRUE;
 631
 632       //cycle through group sets and explicitly add trash param if not set
 633       foreach ($paramByGroup as $setID => $set) {
 634         if (
 635           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 636           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 637         ) {
 638           $this->_params[] = [
 639             'contact_is_deleted',
 640             '=',
 641             '0',
 642             $setID,
 643             '0',
 644           ];
 645         }
 646       }
 647     }
 648
 649     foreach ($this->_params as $value) {
 650       if (empty($value[0])) {
 651         continue;
 652       }
 653       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 654       if ($cfID) {
 655         if (!array_key_exists($cfID, $this->_cfIDs)) {
 656           $this->_cfIDs[$cfID] = [];
 657         }
 658         // Set wildcard value based on "and/or" selection
 659         foreach ($this->_params as $key => $param) {
 660           if ($param[0] == $value[0] . '_operator') {
 661             $value[4] = $param[2] == 'or';
 662             break;
 663           }
 664         }
 665         $this->_cfIDs[$cfID][] = $value;
 666       }
 667
 668       if (!array_key_exists($value[0], $this->_paramLookup)) {
 669         $this->_paramLookup[$value[0]] = [];
 670       }
 671       if ($value[0] !== 'group') {
 672         // Just trying to unravel how group interacts here! This whole function is weird.
 673         $this->_paramLookup[$value[0]][] = $value;
 674       }
 675     }
 676   }
 677
 678   /**
 679    * Some composite fields do not appear in the fields array hack to make them part of the query.
 680    *
 681    * @param $apiEntity
 682    *   The api entity being called.
 683    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 684    */
 685   public function addSpecialFields($apiEntity) {
 686     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 687     // if get called via Contact.get API having address_id as return parameter
 688     if ($apiEntity === 'Contact') {
 689       $special[] = 'address_id';
 690     }
 691     foreach ($special as $name) {
 692       if (!empty($this->_returnProperties[$name])) {
 693         if ($name === 'address_id') {
 694           $this->_tables['civicrm_address'] = 1;
 695           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 696           $this->_element['address_id'] = 1;
 697         }
 698         else {
 699           $this->_select[$name] = "contact_a.{$name} as $name";
 700           $this->_element[$name] = 1;
 701         }
 702       }
 703     }
 704   }
 705
 706   /**
 707    * Given a list of conditions in params and a list of desired
 708    * return Properties generate the required select and from
 709    * clauses. Note that since the where clause introduces new
 710    * tables, the initial attempt also retrieves all variables used
 711    * in the params list
 712    *
 713    * @param string $apiEntity
 714    *   The api entity being called.
 715    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 716    */
 717   public function selectClause($apiEntity = NULL) {
 718
 719     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 720     // param ($apiEntity) instead - presumably an oversight & 2 because
 721     // contact is not implemented as a component.
 722     $this->addSpecialFields($apiEntity);
 723
 724     foreach ($this->_fields as $name => $field) {
 725       // skip component fields
 726       // there are done by the alter query below
 727       // and need not be done on every field
 728       // @todo remove these & handle using metadata - only obscure fields
 729       // that are hack-added should need to be excluded from the main loop.
 730       if (
 731         (substr($name, 0, 12) === 'participant_') ||
 732         (substr($name, 0, 7) === 'pledge_') ||
 733         (substr($name, 0, 5) === 'case_')
 734       ) {
 735         continue;
 736       }
 737
 738       // redirect to activity select clause
 739       if (
 740         (substr($name, 0, 9) === 'activity_') ||
 741         ($name === 'parent_id')
 742       ) {
 743         CRM_Activity_BAO_Query::select($this);
 744       }
 745
 746       // if this is a hierarchical name, we ignore it
 747       $names = explode('-', $name);
 748       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 749         continue;
 750       }
 751
 752       // make an exception for special cases, to add the field in select clause
 753       $makeException = FALSE;
 754
 755       //special handling for groups/tags
 756       if (in_array($name, ['groups', 'tags', 'notes'])
 757         && isset($this->_returnProperties[substr($name, 0, -1)])
 758       ) {
 759         // @todo instead of setting make exception to get us into
 760         // an if clause that has handling for these fields buried with in it
 761         // move the handling to here.
 762         $makeException = TRUE;
 763       }
 764
 765       // since note has 3 different options we need special handling
 766       // note / note_subject / note_body
 767       if ($name === 'notes') {
 768         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 769           if (isset($this->_returnProperties[$noteField])) {
 770             $makeException = TRUE;
 771             break;
 772           }
 773         }
 774       }
 775
 776       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 777       if (
 778         !empty($this->_paramLookup[$name])
 779         || !empty($this->_returnProperties[$name])
 780         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 781         || $makeException
 782       ) {
 783         if ($cfID) {
 784           // add to cfIDs array if not present
 785           if (!array_key_exists($cfID, $this->_cfIDs)) {
 786             $this->_cfIDs[$cfID] = [];
 787           }
 788         }
 789         elseif (isset($field['where'])) {
 790           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 791           if (isset($tableName)) {
 792             if (!empty(self::$_dependencies[$tableName])) {
 793               $this->_tables['civicrm_address'] = 1;
 794               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 795               $this->_element['address_id'] = 1;
 796             }
 797
 798             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 799               $tableName === 'postal_greeting' || $tableName === 'addressee'
 800             ) {
 801               if ($tableName === 'im_provider') {
 802                 CRM_Core_OptionValue::select($this);
 803               }
 804
 805               if (in_array($tableName,
 806                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 807                 $this->_element["{$name}_id"] = 1;
 808                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 809                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 810                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 811                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 812
 813                 if ($tableName === 'email_greeting') {
 814                   // @todo bad join.
 815                   $this->_pseudoConstantsSelect[$name]['join']
 816                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 817                   $this->_pseudoConstantsSelect[$name]['join'] .=
 818                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 819                 }
 820                 elseif ($tableName === 'postal_greeting') {
 821                   // @todo bad join.
 822                   $this->_pseudoConstantsSelect[$name]['join']
 823                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 824                   $this->_pseudoConstantsSelect[$name]['join'] .=
 825                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 826                 }
 827                 elseif ($tableName == 'addressee') {
 828                   // @todo bad join.
 829                   $this->_pseudoConstantsSelect[$name]['join']
 830                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 831                   $this->_pseudoConstantsSelect[$name]['join'] .=
 832                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 833                 }
 834                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 835
 836                 //get display
 837                 $greetField = "{$name}_display";
 838                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 839                 $this->_element[$greetField] = 1;
 840                 //get custom
 841                 $greetField = "{$name}_custom";
 842                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 843                 $this->_element[$greetField] = 1;
 844               }
 845             }
 846             else {
 847               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 848                 $this->_tables[$tableName] = 1;
 849               }
 850
 851               // also get the id of the tableName
 852               $tName = substr($tableName, 8);
 853               if (in_array($tName, ['country', 'state_province', 'county'])) {
 854                 if ($tName == 'state_province') {
 855                   $this->_pseudoConstantsSelect['state_province_name'] = [
 856                     'pseudoField' => "{$tName}",
 857                     'idCol' => "{$tName}_id",
 858                     'bao' => 'CRM_Core_BAO_Address',
 859                     'table' => "civicrm_{$tName}",
 860                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 861                   ];
 862
 863                   $this->_pseudoConstantsSelect[$tName] = [
 864                     'pseudoField' => 'state_province_abbreviation',
 865                     'idCol' => "{$tName}_id",
 866                     'table' => "civicrm_{$tName}",
 867                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 868                   ];
 869                 }
 870                 else {
 871                   $this->_pseudoConstantsSelect[$name] = [
 872                     'pseudoField' => "{$tName}_id",
 873                     'idCol' => "{$tName}_id",
 874                     'bao' => 'CRM_Core_BAO_Address',
 875                     'table' => "civicrm_{$tName}",
 876                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 877                   ];
 878                 }
 879
 880                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 881                 $this->_element["{$tName}_id"] = 1;
 882               }
 883               elseif ($tName != 'contact') {
 884                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 885                 $this->_element["{$tName}_id"] = 1;
 886               }
 887
 888               //special case for phone
 889               if ($name == 'phone') {
 890                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 891                 $this->_element['phone_type_id'] = 1;
 892               }
 893
 894               // if IM then select provider_id also
 895               // to get "IM Service Provider" in a file to be exported, CRM-3140
 896               if ($name == 'im') {
 897                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 898                 $this->_element['provider_id'] = 1;
 899               }
 900
 901               if ($tName == 'contact' && $fieldName == 'organization_name') {
 902                 // special case, when current employer is set for Individual contact
 903                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 904               }
 905               elseif ($tName == 'contact' && $fieldName === 'id') {
 906                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 907               }
 908               elseif (in_array($tName, ['country', 'county'])) {
 909                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 910                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 911               }
 912               elseif ($tName == 'state_province') {
 913                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 914                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 915               }
 916               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 917                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 918                   $this->_select[$name] = "{$field['where']} as `$name`";
 919                 }
 920               }
 921               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 922                 $this->addPseudoconstantFieldToSelect($name);
 923               }
 924               else {
 925                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 926               }
 927               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 928                 $this->_element[$name] = 1;
 929               }
 930             }
 931           }
 932         }
 933         elseif ($name === 'tags') {
 934           //@todo move this handling outside the big IF & ditch $makeException
 935           $this->_useGroupBy = TRUE;
 936           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 937           $this->_element[$name] = 1;
 938           $this->_tables['civicrm_tag'] = 1;
 939           $this->_tables['civicrm_entity_tag'] = 1;
 940         }
 941         elseif ($name === 'groups') {
 942           //@todo move this handling outside the big IF & ditch $makeException
 943           $this->_useGroupBy = TRUE;
 944           // Duplicates will be created here but better to sort them out in php land.
 945           $this->_select[$name] = "
 946             CONCAT_WS(',',
 947             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 948             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 949           )
 950           as `groups`";
 951           $this->_element[$name] = 1;
 952           $this->_tables['civicrm_group_contact'] = 1;
 953           $this->_tables['civicrm_group_contact_cache'] = 1;
 954           $this->_pseudoConstantsSelect["{$name}"] = [
 955             'pseudoField' => "groups",
 956             'idCol' => 'groups',
 957           ];
 958         }
 959         elseif ($name === 'notes') {
 960           //@todo move this handling outside the big IF & ditch $makeException
 961           // if note field is subject then return subject else body of the note
 962           $noteColumn = 'note';
 963           if (isset($noteField) && $noteField === 'note_subject') {
 964             $noteColumn = 'subject';
 965           }
 966
 967           $this->_useGroupBy = TRUE;
 968           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 969           $this->_element[$name] = 1;
 970           $this->_tables['civicrm_note'] = 1;
 971         }
 972         elseif ($name === 'current_employer') {
 973           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 974           $this->_element[$name] = 1;
 975         }
 976       }
 977
 978       if ($cfID && !empty($field['is_search_range'])) {
 979         // this is a custom field with range search enabled, so we better check for two/from values
 980         if (!empty($this->_paramLookup[$name . '_from'])) {
 981           if (!array_key_exists($cfID, $this->_cfIDs)) {
 982             $this->_cfIDs[$cfID] = [];
 983           }
 984           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 985             // search in the cdID array for the same grouping
 986             $fnd = FALSE;
 987             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 988               if ($c[3] == $p[3]) {
 989                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 990                 $fnd = TRUE;
 991               }
 992             }
 993             if (!$fnd) {
 994               $p[2] = ['from' => $p[2]];
 995               $this->_cfIDs[$cfID][] = $p;
 996             }
 997           }
 998         }
 999         if (!empty($this->_paramLookup[$name . '_to'])) {
1000           if (!array_key_exists($cfID, $this->_cfIDs)) {
1001             $this->_cfIDs[$cfID] = [];
1002           }
1003           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
1004             // search in the cdID array for the same grouping
1005             $fnd = FALSE;
1006             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
1007               if ($c[4] == $p[4]) {
1008                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1009                 $fnd = TRUE;
1010               }
1011             }
1012             if (!$fnd) {
1013               $p[2] = ['to' => $p[2]];
1014               $this->_cfIDs[$cfID][] = $p;
1015             }
1016           }
1017         }
1018       }
1019     }
1020
1021     // add location as hierarchical elements
1022     $this->addHierarchicalElements();
1023
1024     // add multiple field like website
1025     $this->addMultipleElements();
1026
1027     //fix for CRM-951
1028     CRM_Core_Component::alterQuery($this, 'select');
1029
1030     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1031   }
1032
1033   /**
1034    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1035    */
1036   public function addHierarchicalElements() {
1037     if (empty($this->_returnProperties['location'])) {
1038       return;
1039     }
1040     if (!is_array($this->_returnProperties['location'])) {
1041       return;
1042     }
1043
1044     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1045     $processed = [];
1046     $index = 0;
1047
1048     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1049     $addressCustomFieldIds = [];
1050
1051     foreach ($this->_returnProperties['location'] as $name => $elements) {
1052       $lCond = self::getPrimaryCondition($name);
1053       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1054
1055       if (!$lCond) {
1056         if ($locationTypeId === FALSE) {
1057           continue;
1058         }
1059         $lCond = "location_type_id = $locationTypeId";
1060         $this->_useDistinct = TRUE;
1061
1062         //commented for CRM-3256
1063         $this->_useGroupBy = TRUE;
1064       }
1065
1066       $name = str_replace(' ', '_', $name);
1067       $tName = "$name-location_type";
1068       $ltName = "`$name-location_type`";
1069       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1070       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1071       $this->_element["{$tName}_id"] = 1;
1072       $this->_element["{$tName}"] = 1;
1073
1074       $locationTypeName = $tName;
1075       $locationTypeJoin = [];
1076
1077       $addWhereCount = 0;
1078       foreach ($elements as $elementFullName => $dontCare) {
1079         $index++;
1080         $elementName = $elementCmpName = $elementFullName;
1081
1082         if (substr($elementCmpName, 0, 5) == 'phone') {
1083           $elementCmpName = 'phone';
1084         }
1085
1086         if (in_array($elementCmpName, array_keys($addressCustomFields))) {
1087           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1088             $addressCustomFieldIds[$cfID][$name] = 1;
1089           }
1090         }
1091         // add address table - doesn't matter if we do it mutliple times - it's the same data
1092         // @todo ditch the double processing of addressJoin
1093         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1094           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1095         ) {
1096           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1097           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1098           $processed[$aName] = 1;
1099         }
1100
1101         $cond = $elementType = '';
1102         if (strpos($elementName, '-') !== FALSE) {
1103           // this is either phone, email or IM
1104           list($elementName, $elementType) = explode('-', $elementName);
1105
1106           if (($elementName != 'phone') && ($elementName != 'im')) {
1107             $cond = self::getPrimaryCondition($elementType);
1108           }
1109           // CRM-13011 : If location type is primary, do not restrict search to the phone
1110           // type id - we want the primary phone, regardless of what type it is.
1111           // Otherwise, restrict to the specified phone type for the given field.
1112           if ((!$cond) && ($elementName == 'phone')) {
1113             $cond = "phone_type_id = '$elementType'";
1114           }
1115           elseif ((!$cond) && ($elementName == 'im')) {
1116             // IM service provider id, CRM-3140
1117             $cond = "provider_id = '$elementType'";
1118           }
1119           $elementType = '-' . $elementType;
1120         }
1121
1122         $field = $this->_fields[$elementName] ?? NULL;
1123
1124         // hack for profile, add location id
1125         if (!$field) {
1126           if ($elementType &&
1127             // fix for CRM-882( to handle phone types )
1128             !is_numeric($elementType)
1129           ) {
1130             if (is_numeric($name)) {
1131               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1132             }
1133             else {
1134               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1135             }
1136           }
1137           elseif (is_numeric($name)) {
1138             //this for phone type to work
1139             if (in_array($elementName, ['phone', 'phone_ext'])) {
1140               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1141             }
1142             else {
1143               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1144             }
1145           }
1146           else {
1147             //this is for phone type to work for profile edit
1148             if (in_array($elementName, ['phone', 'phone_ext'])) {
1149               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1150             }
1151             else {
1152               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1153             }
1154           }
1155         }
1156
1157         // Check if there is a value, if so also add to where Clause
1158         $addWhere = FALSE;
1159         if ($this->_params) {
1160           $nm = $elementName;
1161           if (isset($locationTypeId)) {
1162             $nm .= "-$locationTypeId";
1163           }
1164           if (!is_numeric($elementType)) {
1165             $nm .= "$elementType";
1166           }
1167
1168           foreach ($this->_params as $id => $values) {
1169             if ((is_array($values) && $values[0] == $nm) ||
1170               (in_array($elementName, ['phone', 'im'])
1171                 && (strpos($values[0], $nm) !== FALSE)
1172               )
1173             ) {
1174               $addWhere = TRUE;
1175               $addWhereCount++;
1176               break;
1177             }
1178           }
1179         }
1180
1181         if ($field && isset($field['where'])) {
1182           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1183           $pf = substr($tableName, 8);
1184           $tName = $name . '-' . $pf . $elementType;
1185           if (isset($tableName)) {
1186             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1187               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1188             }
1189             else {
1190               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1191             }
1192
1193             $this->_element["{$tName}_id"] = 1;
1194             if (substr($tName, -15) == '-state_province') {
1195               // FIXME: hack to fix CRM-1900
1196               $a = Civi::settings()->get('address_format');
1197
1198               if (substr_count($a, 'state_province_name') > 0) {
1199                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1200                   'pseudoField' => "{$pf}_id",
1201                   'idCol' => "{$tName}_id",
1202                   'bao' => 'CRM_Core_BAO_Address',
1203                 ];
1204                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1205               }
1206               else {
1207                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1208                   'pseudoField' => 'state_province_abbreviation',
1209                   'idCol' => "{$tName}_id",
1210                 ];
1211                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1212               }
1213             }
1214             else {
1215               if (substr($elementFullName, 0, 2) == 'im') {
1216                 $provider = "{$name}-{$elementFullName}-provider_id";
1217                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1218                 $this->_element[$provider] = 1;
1219               }
1220               if ($pf == 'country' || $pf == 'county') {
1221                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1222                   'pseudoField' => "{$pf}_id",
1223                   'idCol' => "{$tName}_id",
1224                   'bao' => 'CRM_Core_BAO_Address',
1225                 ];
1226                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1227               }
1228               else {
1229                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1230               }
1231             }
1232
1233             if (in_array($pf, ['state_province', 'country', 'county'])) {
1234               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1235             }
1236             else {
1237               $this->_element["{$name}-{$elementFullName}"] = 1;
1238             }
1239
1240             if (empty($processed["`$tName`"])) {
1241               $processed["`$tName`"] = 1;
1242               $newName = $tableName . '_' . $index;
1243               switch ($tableName) {
1244                 case 'civicrm_phone':
1245                 case 'civicrm_email':
1246                 case 'civicrm_im':
1247                 case 'civicrm_openid':
1248
1249                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1250                   if ($tableName != 'civicrm_phone') {
1251                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1252                   }
1253                   elseif (is_numeric($name)) {
1254                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1255                   }
1256
1257                   // this special case to add phone type
1258                   if ($cond) {
1259                     $phoneTypeCondition = " AND `$tName`.$cond ";
1260                     //gross hack to pickup corrupted data also, CRM-7603
1261                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1262                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1263                       if (!empty($lCond)) {
1264                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1265                       }
1266                     }
1267                     $this->_tables[$tName] .= $phoneTypeCondition;
1268                   }
1269
1270                   //build locationType join
1271                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1272
1273                   if ($addWhere) {
1274                     $this->_whereTables[$tName] = $this->_tables[$tName];
1275                   }
1276                   break;
1277
1278                 case 'civicrm_state_province':
1279                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1280                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1281                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1282                   if ($addWhere) {
1283                     $this->_whereTables["{$name}-address"] = $addressJoin;
1284                   }
1285                   break;
1286
1287                 case 'civicrm_country':
1288                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1289                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1290                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1291                   if ($addWhere) {
1292                     $this->_whereTables["{$name}-address"] = $addressJoin;
1293                   }
1294                   break;
1295
1296                 case 'civicrm_county':
1297                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1298                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1299                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1300                   if ($addWhere) {
1301                     $this->_whereTables["{$name}-address"] = $addressJoin;
1302                   }
1303                   break;
1304
1305                 default:
1306                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1307                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1308                   }
1309                   if ($addWhere) {
1310                     $this->_whereTables["{$name}-address"] = $addressJoin;
1311                   }
1312                   break;
1313               }
1314             }
1315           }
1316         }
1317       }
1318
1319       // add location type  join
1320       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1321       $this->_tables[$locationTypeName] = $ltypeJoin;
1322
1323       // table should be present in $this->_whereTables,
1324       // to add its condition in location type join, CRM-3939.
1325       if ($addWhereCount) {
1326         $locClause = [];
1327         foreach ($this->_whereTables as $tableName => $clause) {
1328           if (!empty($locationTypeJoin[$tableName])) {
1329             $locClause[] = $locationTypeJoin[$tableName];
1330           }
1331         }
1332
1333         if (!empty($locClause)) {
1334           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1335         }
1336       }
1337     }
1338
1339     if (!empty($addressCustomFieldIds)) {
1340       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1341       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1342         foreach ($locTypeName as $name => $dnc) {
1343           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1344           $fieldName = "$name-custom_{$cfID}";
1345           $tName = "$name-address-custom-{$cfID}";
1346           $aName = "`$name-address-custom-{$cfID}`";
1347           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1348           $this->_element["{$tName}_id"] = 1;
1349           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1350           $this->_element[$fieldName] = 1;
1351           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1352         }
1353       }
1354     }
1355   }
1356
1357   /**
1358    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1359    */
1360   public function addMultipleElements() {
1361     if (empty($this->_returnProperties['website'])) {
1362       return;
1363     }
1364     if (!is_array($this->_returnProperties['website'])) {
1365       return;
1366     }
1367
1368     foreach ($this->_returnProperties['website'] as $key => $elements) {
1369       foreach ($elements as $elementFullName => $dontCare) {
1370         $tName = "website-{$key}-{$elementFullName}";
1371         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1372         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1373         $this->_element["{$tName}_id"] = 1;
1374         $this->_element["{$tName}"] = 1;
1375
1376         $type = "website-{$key}-website_type_id";
1377         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1378         $this->_element[$type] = 1;
1379         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1380       }
1381     }
1382   }
1383
1384   /**
1385    * Generate the query based on what type of query we need.
1386    *
1387    * @param bool $count
1388    * @param bool $sortByChar
1389    * @param bool $groupContacts
1390    * @param bool $onlyDeleted
1391    *
1392    * @return array
1393    *   sql query parts as an array
1394    */
1395   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1396     // build permission clause
1397     $this->generatePermissionClause($onlyDeleted, $count);
1398
1399     if ($count) {
1400       if (isset($this->_rowCountClause)) {
1401         $select = "SELECT {$this->_rowCountClause}";
1402       }
1403       elseif (isset($this->_distinctComponentClause)) {
1404         // we add distinct to get the right count for components
1405         // for the more complex result set, we use GROUP BY the same id
1406         // CRM-9630
1407         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1408       }
1409       else {
1410         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1411       }
1412       $from = $this->_simpleFromClause;
1413       if ($this->_useDistinct) {
1414         $this->_useGroupBy = TRUE;
1415       }
1416     }
1417     elseif ($sortByChar) {
1418       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1419       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1420       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1421       $from = $this->_simpleFromClause;
1422     }
1423     elseif ($groupContacts) {
1424       $select = 'SELECT contact_a.id as id';
1425       if ($this->_useDistinct) {
1426         $this->_useGroupBy = TRUE;
1427       }
1428       $from = $this->_simpleFromClause;
1429     }
1430     else {
1431       if (!empty($this->_paramLookup['group'])) {
1432
1433         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1434
1435         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1436           $this->_paramLookup['group'][0][1] = key($value);
1437         }
1438
1439         // Presumably the lines below come into manage groups screen.
1440         // make sure there is only one element
1441         // this is used when we are running under smog and need to know
1442         // how the contact was added (CRM-1203)
1443         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1444         if ((count($this->_paramLookup['group']) == 1) &&
1445           (count($groups) == 1)
1446         ) {
1447           $groupId = $groups[0];
1448
1449           //check if group is saved search
1450           $group = new CRM_Contact_BAO_Group();
1451           $group->id = $groupId;
1452           $group->find(TRUE);
1453
1454           if (!isset($group->saved_search_id)) {
1455             $tbName = "civicrm_group_contact";
1456             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1457             // as this will add load to an intentionally light query.
1458             // ideally this code would be removed as it appears to be to support CRM-1203
1459             // and passing in the required returnProperties from the url would
1460             // make more sense that globally applying the requirements of one form.
1461             if (($this->_returnProperties != ['contact_id'])) {
1462               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1463               $this->_element['group_contact_id'] = 1;
1464               $this->_select['status'] = "$tbName.status as status";
1465               $this->_element['status'] = 1;
1466             }
1467           }
1468         }
1469         $this->_useGroupBy = TRUE;
1470       }
1471       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1472         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1473           // CRM-5954
1474           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1475           $this->_useDistinct = FALSE;
1476           $this->_useGroupBy = TRUE;
1477         }
1478       }
1479
1480       $select = $this->getSelect();
1481       $from = $this->_fromClause;
1482     }
1483
1484     $where = '';
1485     if (!empty($this->_whereClause)) {
1486       $where = "WHERE {$this->_whereClause}";
1487     }
1488
1489     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1490       if (empty($where)) {
1491         $where = "WHERE $this->_permissionWhereClause";
1492       }
1493       else {
1494         $where = "$where AND $this->_permissionWhereClause";
1495       }
1496     }
1497
1498     $having = '';
1499     if (!empty($this->_having)) {
1500       foreach ($this->_having as $havingSets) {
1501         foreach ($havingSets as $havingSet) {
1502           $havingValue[] = $havingSet;
1503         }
1504       }
1505       $having = ' HAVING ' . implode(' AND ', $havingValue);
1506     }
1507
1508     // if we are doing a transform, do it here
1509     // use the $from, $where and $having to get the contact ID
1510     if ($this->_displayRelationshipType) {
1511       $this->filterRelatedContacts($from, $where, $having);
1512     }
1513
1514     return [$select, $from, $where, $having];
1515   }
1516
1517   /**
1518    * Get where values from the parameters.
1519    *
1520    * @param string $name
1521    * @param mixed $grouping
1522    *
1523    * @return mixed
1524    */
1525   public function getWhereValues($name, $grouping) {
1526     $result = NULL;
1527     foreach ($this->_params as $values) {
1528       if ($values[0] == $name && $values[3] == $grouping) {
1529         return $values;
1530       }
1531     }
1532
1533     return $result;
1534   }
1535
1536   /**
1537    * Fix date values.
1538    *
1539    * @param bool $relative
1540    * @param string $from
1541    * @param string $to
1542    */
1543   public static function fixDateValues($relative, &$from, &$to) {
1544     if ($relative) {
1545       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1546     }
1547   }
1548
1549   /**
1550    * Convert values from form-appropriate to query-object appropriate.
1551    *
1552    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1553    * So, ideally we would convert all fields to look like
1554    *  array(
1555    *   0 => $fieldName
1556    *   // Set the operator for legacy reasons, but it is ignored
1557    *   1 =>  '='
1558    *   // array in sql filter syntax
1559    *   2 => array('BETWEEN' => array(1,60),
1560    *   3 => null
1561    *   4 => null
1562    *  );
1563    *
1564    * There are some examples of the syntax in
1565    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1566    *
1567    * More notes at CRM_Core_DAO::createSQLFilter
1568    *
1569    * and a list of supported operators in CRM_Core_DAO
1570    *
1571    * @param array $formValues
1572    * @param int $wildcard
1573    * @param bool $useEquals
1574    *
1575    * @param string $apiEntity
1576    *
1577    * @param array $entityReferenceFields
1578    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1579    *
1580    * @return array
1581    */
1582   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1583     $entityReferenceFields = []) {
1584     $params = [];
1585     if (empty($formValues)) {
1586       return $params;
1587     }
1588
1589     self::filterCountryFromValuesIfStateExists($formValues);
1590
1591     foreach ($formValues as $id => $values) {
1592       if (self::isAlreadyProcessedForQueryFormat($values)) {
1593         $params[] = $values;
1594         continue;
1595       }
1596
1597       self::legacyConvertFormValues($id, $values);
1598
1599       // The form uses 1 field to represent two db fields
1600       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1601         $contactType = [];
1602         $subType = [];
1603         foreach ((array) $values as $key => $type) {
1604           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1605           $contactType[$types[0]] = $types[0];
1606           // Add sub-type if specified
1607           if (!empty($types[1])) {
1608             $subType[$types[1]] = $types[1];
1609           }
1610         }
1611         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1612         if ($subType) {
1613           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1614         }
1615       }
1616       elseif ($id === 'privacy') {
1617         if (is_array($formValues['privacy'])) {
1618           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1619           foreach ($formValues['privacy'] as $key => $value) {
1620             if ($value) {
1621               $params[] = [$key, $op, $value, 0, 0];
1622             }
1623           }
1624         }
1625       }
1626       elseif ($id === 'email_on_hold') {
1627         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1628           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1629           // so in 5.11 we have an extra if that should become redundant over time.
1630           // https://lab.civicrm.org/dev/core/issues/745
1631           // @todo this renaming of email_on_hold to on_hold needs revisiting
1632           // it precedes recent changes but causes the default not to reload.
1633           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1634           if (!empty($onHoldValue)) {
1635             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1636           }
1637         }
1638       }
1639       elseif (substr($id, 0, 7) === 'custom_'
1640         &&  (
1641           substr($id, -5, 5) === '_from'
1642           || substr($id, -3, 3) === '_to'
1643         )
1644       ) {
1645         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1646       }
1647       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1648         FALSE)) {
1649         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1650       }
1651       else {
1652         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1653
1654         if (!$values) {
1655           continue;
1656         }
1657         $params[] = $values;
1658       }
1659     }
1660     return $params;
1661   }
1662
1663   /**
1664    * Function to support legacy format for groups and tags.
1665    *
1666    * @param string $id
1667    * @param array|int $values
1668    *
1669    */
1670   public static function legacyConvertFormValues($id, &$values) {
1671     $legacyElements = [
1672       'group',
1673       'tag',
1674       'contact_tags',
1675       'contact_type',
1676       'membership_type_id',
1677       'membership_status_id',
1678     ];
1679     if (in_array($id, $legacyElements) && is_array($values)) {
1680       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1681       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1682       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1683     }
1684   }
1685
1686   /**
1687    * Fix values from query from/to something no-one cared enough to document.
1688    *
1689    * @param int $id
1690    * @param array $values
1691    * @param int $wildcard
1692    * @param bool $useEquals
1693    *
1694    * @param string $apiEntity
1695    *
1696    * @return array|null
1697    */
1698   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1699     // skip a few search variables
1700     static $skipWhere = NULL;
1701     static $likeNames = NULL;
1702     $result = NULL;
1703
1704     // Change camelCase EntityName to lowercase with underscores
1705     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1706
1707     // check if $value is in OK (Operator as Key) format as used by Get API
1708     if (CRM_Utils_System::isNull($values)) {
1709       return $result;
1710     }
1711
1712     if (!$skipWhere) {
1713       $skipWhere = [
1714         'task',
1715         'radio_ts',
1716         'uf_group_id',
1717         'component_mode',
1718         'qfKey',
1719         'operator',
1720         'display_relationship_type',
1721       ];
1722     }
1723
1724     if (in_array($id, $skipWhere) ||
1725       substr($id, 0, 4) == '_qf_' ||
1726       substr($id, 0, 7) == 'hidden_'
1727     ) {
1728       return $result;
1729     }
1730
1731     if ($apiEntity &&
1732       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1733       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1734       (substr($id, 0, 7) != 'custom_')
1735     ) {
1736       $id = $apiEntity . '_' . $id;
1737     }
1738
1739     if (!$likeNames) {
1740       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1741     }
1742
1743     // email comes in via advanced search
1744     // so use wildcard always
1745     if ($id == 'email') {
1746       $wildcard = 1;
1747     }
1748
1749     if (!$useEquals && in_array($id, $likeNames)) {
1750       $result = [$id, 'LIKE', $values, 0, 1];
1751     }
1752     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1753       $result = [$id, 'LIKE', $values, 0, 0];
1754     }
1755     elseif ($id == 'contact_type' ||
1756       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1757     ) {
1758       $result = [$id, 'IN', $values, 0, $wildcard];
1759     }
1760     else {
1761       $result = [$id, '=', $values, 0, $wildcard];
1762     }
1763
1764     return $result;
1765   }
1766
1767   /**
1768    * Get the where clause for a single field.
1769    *
1770    * @param array $values
1771    * @param bool $isForcePrimaryOnly
1772    *
1773    * @throws \CRM_Core_Exception
1774    */
1775   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1776     if ($this->isARelativeDateField($values[0])) {
1777       $this->buildRelativeDateQuery($values);
1778       return;
1779     }
1780     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1781
1782     // do not process custom fields or prefixed contact ids or component params
1783     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1784       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1785       (substr($values[0], 0, 13) === 'contribution_') ||
1786       (substr($values[0], 0, 6) === 'event_') ||
1787       (substr($values[0], 0, 12) === 'participant_') ||
1788       (substr($values[0], 0, 7) === 'member_') ||
1789       (substr($values[0], 0, 6) === 'grant_') ||
1790       (substr($values[0], 0, 7) === 'pledge_') ||
1791       (substr($values[0], 0, 5) === 'case_') ||
1792       (substr($values[0], 0, 10) === 'financial_') ||
1793       (substr($values[0], 0, 8) === 'payment_') ||
1794       (substr($values[0], 0, 11) === 'membership_')
1795       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1796       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1797       // be needed.
1798       || $values[0] === 'mailing_id'
1799     ) {
1800       return;
1801     }
1802
1803     // skip for hook injected fields / params
1804     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1805     if (array_key_exists($values[0], $extFields)) {
1806       return;
1807     }
1808
1809     switch ($values[0]) {
1810       case 'deleted_contacts':
1811         $this->deletedContacts($values);
1812         return;
1813
1814       case 'contact_sub_type':
1815         $this->contactSubType($values);
1816         return;
1817
1818       case 'group':
1819       case 'group_type':
1820         $this->group($values);
1821         return;
1822
1823       // case tag comes from find contacts
1824       case 'tag_search':
1825         $this->tagSearch($values);
1826         return;
1827
1828       case 'tag':
1829       case 'contact_tags':
1830         $this->tag($values);
1831         return;
1832
1833       case 'note':
1834       case 'note_body':
1835       case 'note_subject':
1836         $this->notes($values);
1837         return;
1838
1839       case 'uf_user':
1840         $this->ufUser($values);
1841         return;
1842
1843       case 'sort_name':
1844       case 'display_name':
1845         $this->sortName($values);
1846         return;
1847
1848       case 'addressee':
1849       case 'postal_greeting':
1850       case 'email_greeting':
1851         $this->greetings($values);
1852         return;
1853
1854       case 'email':
1855       case 'email_id':
1856         $this->email($values, $isForcePrimaryOnly);
1857         return;
1858
1859       case 'phone_numeric':
1860         $this->phone_numeric($values);
1861         return;
1862
1863       case 'phone_phone_type_id':
1864       case 'phone_location_type_id':
1865         $this->phone_option_group($values);
1866         return;
1867
1868       case 'street_address':
1869         $this->street_address($values);
1870         return;
1871
1872       case 'street_number':
1873         $this->street_number($values);
1874         return;
1875
1876       case 'sortByCharacter':
1877         $this->sortByCharacter($values);
1878         return;
1879
1880       case 'location_type':
1881         $this->locationType($values);
1882         return;
1883
1884       case 'county':
1885         $this->county($values);
1886         return;
1887
1888       case 'state_province':
1889       case 'state_province_id':
1890       case 'state_province_name':
1891         $this->stateProvince($values);
1892         return;
1893
1894       case 'country':
1895       case 'country_id':
1896         $this->country($values, FALSE);
1897         return;
1898
1899       case 'postal_code':
1900       case 'postal_code_low':
1901       case 'postal_code_high':
1902         $this->postalCode($values);
1903         return;
1904
1905       case 'activity_date':
1906       case 'activity_date_low':
1907       case 'activity_date_high':
1908       case 'activity_date_time_low':
1909       case 'activity_date_time_high':
1910       case 'activity_role':
1911       case 'activity_status_id':
1912       case 'activity_status':
1913       case 'activity_priority':
1914       case 'activity_priority_id':
1915       case 'followup_parent_id':
1916       case 'parent_id':
1917       case 'source_contact_id':
1918       case 'activity_text':
1919       case 'activity_option':
1920       case 'test_activities':
1921       case 'activity_type_id':
1922       case 'activity_type':
1923       case 'activity_survey_id':
1924       case 'activity_tags':
1925       case 'activity_taglist':
1926       case 'activity_test':
1927       case 'activity_campaign_id':
1928       case 'activity_engagement_level':
1929       case 'activity_id':
1930       case 'activity_result':
1931       case 'source_contact':
1932         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1933         return;
1934
1935       case 'age_low':
1936       case 'age_high':
1937       case 'birth_date_low':
1938       case 'birth_date_high':
1939       case 'deceased_date_low':
1940       case 'deceased_date_high':
1941         $this->demographics($values);
1942         return;
1943
1944       case 'age_asof_date':
1945         // handled by demographics
1946         return;
1947
1948       case 'log_date_low':
1949       case 'log_date_high':
1950         $this->modifiedDates($values);
1951         return;
1952
1953       case 'changed_by':
1954         $this->changeLog($values);
1955         return;
1956
1957       case 'do_not_phone':
1958       case 'do_not_email':
1959       case 'do_not_mail':
1960       case 'do_not_sms':
1961       case 'do_not_trade':
1962       case 'is_opt_out':
1963         $this->privacy($values);
1964         return;
1965
1966       case 'privacy_options':
1967         $this->privacyOptions($values);
1968         return;
1969
1970       case 'privacy_operator':
1971       case 'privacy_toggle':
1972         // these are handled by privacy options
1973         return;
1974
1975       case 'preferred_communication_method':
1976         $this->preferredCommunication($values);
1977         return;
1978
1979       case 'relation_type_id':
1980       case 'relationship_start_date_high':
1981       case 'relationship_start_date_low':
1982       case 'relationship_end_date_high':
1983       case 'relationship_end_date_low':
1984       case 'relation_active_period_date_high':
1985       case 'relation_active_period_date_low':
1986       case 'relation_target_name':
1987       case 'relation_status':
1988       case 'relation_description':
1989       case 'relation_date_low':
1990       case 'relation_date_high':
1991         $this->relationship($values);
1992         $this->_relationshipValuesAdded = TRUE;
1993         return;
1994
1995       case 'task_status_id':
1996         $this->task($values);
1997         return;
1998
1999       case 'task_id':
2000         // since this case is handled with the above
2001         return;
2002
2003       case 'prox_distance':
2004         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2005         return;
2006
2007       case 'prox_street_address':
2008       case 'prox_city':
2009       case 'prox_postal_code':
2010       case 'prox_state_province_id':
2011       case 'prox_country_id':
2012       case 'prox_geo_code_1':
2013       case 'prox_geo_code_2':
2014         // handled by the proximity_distance clause
2015         return;
2016
2017       default:
2018         $this->restWhere($values);
2019         return;
2020     }
2021   }
2022
2023   /**
2024    * Given a list of conditions in params generate the required where clause.
2025    *
2026    * @param bool $isForcePrimaryEmailOnly
2027    *
2028    * @return string
2029    * @throws \CRM_Core_Exception
2030    */
2031   public function whereClause($isForcePrimaryEmailOnly = NULL) {
2032     $this->_where[0] = [];
2033     $this->_qill[0] = [];
2034
2035     $this->includeContactIDs();
2036     if (!empty($this->_params)) {
2037       foreach (array_keys($this->_params) as $id) {
2038         if (empty($this->_params[$id][0])) {
2039           continue;
2040         }
2041         // check for both id and contact_id
2042         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2043           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2044           $field = $this->_fields['id'] ?? NULL;
2045           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2046             'CRM_Contact_BAO_Contact',
2047             "contact_a.id",
2048             $this->_params[$id][2],
2049             $this->_params[$id][1]
2050           );
2051           $this->_qill[0][] = ts("%1 %2 %3", [
2052             1 => $field['title'] ?? '',
2053             2 => $qillop,
2054             3 => $qillVal,
2055           ]);
2056         }
2057         else {
2058           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2059         }
2060       }
2061
2062       CRM_Core_Component::alterQuery($this, 'where');
2063
2064       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2065     }
2066
2067     if ($this->_customQuery) {
2068       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2069       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2070       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2071       if (!empty($this->_customQuery->_where)) {
2072         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2073       }
2074       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2075     }
2076
2077     $clauses = [];
2078     $andClauses = [];
2079
2080     $validClauses = 0;
2081     if (!empty($this->_where)) {
2082       foreach ($this->_where as $grouping => $values) {
2083         if ($grouping > 0 && !empty($values)) {
2084           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2085           $validClauses++;
2086         }
2087       }
2088
2089       if (!empty($this->_where[0])) {
2090         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2091       }
2092       if (!empty($clauses)) {
2093         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2094       }
2095
2096       if ($validClauses > 1) {
2097         $this->_useDistinct = TRUE;
2098       }
2099     }
2100
2101     return implode(' AND ', $andClauses);
2102   }
2103
2104   /**
2105    * Generate where clause for any parameters not already handled.
2106    *
2107    * @param array $values
2108    *
2109    * @throws Exception
2110    */
2111   public function restWhere(&$values) {
2112     $name = $values[0] ?? NULL;
2113     $op = $values[1] ?? NULL;
2114     $value = $values[2] ?? NULL;
2115     $grouping = $values[3] ?? NULL;
2116     $wildcard = $values[4] ?? NULL;
2117
2118     if (isset($grouping) && empty($this->_where[$grouping])) {
2119       $this->_where[$grouping] = [];
2120     }
2121
2122     $multipleFields = ['url'];
2123
2124     //check if the location type exists for fields
2125     $lType = '';
2126     $locType = explode('-', $name);
2127
2128     if (!in_array($locType[0], $multipleFields)) {
2129       //add phone type if exists
2130       if (isset($locType[2]) && $locType[2]) {
2131         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2132       }
2133     }
2134
2135     $field = $this->_fields[$name] ?? NULL;
2136
2137     if (!$field) {
2138       $field = $this->_fields[$locType[0]] ?? NULL;
2139
2140       if (!$field) {
2141         // Strip any trailing _high & _low that might be appended.
2142         $realFieldName = str_replace(['_high', '_low'], '', $name);
2143         if (isset($this->_fields[$realFieldName])) {
2144           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2145           $columnName = $field['column_name'] ?? $field['name'];
2146           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2147         }
2148         return;
2149       }
2150     }
2151
2152     $setTables = TRUE;
2153
2154     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2155     if (isset($locType[1]) && is_numeric($locType[1])) {
2156       $lType = $locationType[$locType[1]];
2157     }
2158     if ($lType) {
2159       $field['title'] .= " ($lType)";
2160     }
2161
2162     if (substr($name, 0, 14) === 'state_province') {
2163       if (isset($locType[1]) && is_numeric($locType[1])) {
2164         $setTables = FALSE;
2165         $aName = "{$lType}-address";
2166         $where = "`$aName`.state_province_id";
2167       }
2168       else {
2169         $where = "civicrm_address.state_province_id";
2170       }
2171
2172       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2173       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2174       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2175       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2176     }
2177     elseif (!empty($field['pseudoconstant'])) {
2178       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2179       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2180       $this->optionValueQuery(
2181         $name, $op, $value, $grouping,
2182         'CRM_Contact_DAO_Contact',
2183         $field,
2184         $field['title'],
2185         CRM_Utils_Type::typeToString($dataType)
2186       );
2187       if ($name === 'gender_id') {
2188         self::$_openedPanes[ts('Demographics')] = TRUE;
2189       }
2190     }
2191     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2192       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2193       if (isset($locType[1]) && is_numeric($locType[1])) {
2194         $setTables = FALSE;
2195         $aName = "{$lType}-address";
2196         $where = "`$aName`.$name";
2197       }
2198       else {
2199         $where = "civicrm_address.$name";
2200       }
2201
2202       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2203       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2204
2205       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2206       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2207     }
2208     elseif ($name === 'world_region') {
2209       $this->optionValueQuery(
2210         $name, $op, $value, $grouping,
2211         NULL,
2212         $field,
2213         ts('World Region'),
2214         'Positive'
2215       );
2216     }
2217     elseif ($name === 'is_deceased') {
2218       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2219       self::$_openedPanes[ts('Demographics')] = TRUE;
2220     }
2221     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2222       $appendDateTime = TRUE;
2223       if ($name === 'deceased_date' || $name === 'birth_date') {
2224         $appendDateTime = FALSE;
2225         self::$_openedPanes[ts('Demographics')] = TRUE;
2226       }
2227       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2228     }
2229     elseif ($name === 'contact_id') {
2230       if (is_int($value)) {
2231         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2232         $this->_qill[$grouping][] = "$field[title] $op $value";
2233       }
2234     }
2235     elseif ($name === 'name') {
2236       $value = CRM_Core_DAO::escapeString($value);
2237       if ($wildcard) {
2238         $op = 'LIKE';
2239         $value = self::getWildCardedValue($wildcard, $op, $value);
2240       }
2241       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2242       // @todo it's likely this code path is obsolete / never called. It is definitely not
2243       // passed through in our test suite.
2244       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2245       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2246     }
2247     elseif ($name === 'current_employer') {
2248       if ($wildcard) {
2249         $op = 'LIKE';
2250         $value = self::getWildCardedValue($wildcard, $op, $value);
2251       }
2252       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2253         $value
2254       );
2255       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2256       $this->_where[$grouping][] = $ceWhereClause;
2257       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2258     }
2259     elseif (substr($name, 0, 4) === 'url-') {
2260       $tName = 'civicrm_website';
2261       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2262       $value = CRM_Core_DAO::escapeString($value);
2263       if ($wildcard) {
2264         $op = 'LIKE';
2265         $value = self::getWildCardedValue($wildcard, $op, $value);
2266       }
2267
2268       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2269       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2270     }
2271     elseif ($name === 'contact_is_deleted') {
2272       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2273     }
2274     elseif (!empty($field['where'])) {
2275       $type = NULL;
2276       if (!empty($field['type'])) {
2277         $type = CRM_Utils_Type::typeToString($field['type']);
2278       }
2279
2280       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2281
2282       if (isset($locType[1]) &&
2283         is_numeric($locType[1])
2284       ) {
2285         $setTables = FALSE;
2286
2287         //get the location name
2288         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2289         $fieldName = "`$tName`.$fldName";
2290
2291         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2292         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2293
2294       }
2295       else {
2296         if ($tableName == 'civicrm_contact') {
2297           $fieldName = "contact_a.{$fieldName}";
2298         }
2299         else {
2300           $fieldName = $field['where'];
2301         }
2302       }
2303
2304       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2305       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2306         1 => $field['title'],
2307         2 => $qillop,
2308         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2309       ]);
2310
2311       if (is_array($value)) {
2312         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2313         // for api style operators for functions that hit this point without worrying about regression
2314         // (the previous comments indicated the condition for hitting this point were unknown
2315         // per CRM-14743 we are adding modified_date & created_date operator support
2316         $operations = array_keys($value);
2317         foreach ($operations as $operator) {
2318           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2319             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2320             //so this condition will satisfy the search for now
2321             if (strpos($op, 'IN') !== FALSE) {
2322               $value = [$op => $value];
2323             }
2324             // we don't know when this might happen
2325             else {
2326               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2327             }
2328           }
2329         }
2330         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2331       }
2332       else {
2333         if ($wildcard) {
2334           $op = 'LIKE';
2335           $value = self::getWildCardedValue($wildcard, $op, $value);
2336         }
2337
2338         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2339       }
2340     }
2341
2342     if ($setTables && isset($field['where'])) {
2343       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2344       if (isset($tableName)) {
2345         $this->_tables[$tableName] = 1;
2346         $this->_whereTables[$tableName] = 1;
2347       }
2348     }
2349   }
2350
2351   /**
2352    * @param $where
2353    * @param $locType
2354    *
2355    * @return array
2356    * @throws Exception
2357    */
2358   public static function getLocationTableName(&$where, &$locType) {
2359     if (isset($locType[1]) && is_numeric($locType[1])) {
2360       list($tbName, $fldName) = explode(".", $where);
2361
2362       //get the location name
2363       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2364       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2365       if (in_array($locType[0], $specialFields)) {
2366         //hack to fix / special handing for phone_ext
2367         if ($locType[0] == 'phone_ext') {
2368           $locType[0] = 'phone';
2369         }
2370         if (isset($locType[2]) && $locType[2]) {
2371           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2372         }
2373         else {
2374           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2375         }
2376       }
2377       elseif (in_array($locType[0],
2378         [
2379           'address_name',
2380           'street_address',
2381           'street_name',
2382           'street_number_suffix',
2383           'street_unit',
2384           'supplemental_address_1',
2385           'supplemental_address_2',
2386           'supplemental_address_3',
2387           'city',
2388           'postal_code',
2389           'postal_code_suffix',
2390           'geo_code_1',
2391           'geo_code_2',
2392           'master_id',
2393         ]
2394       )) {
2395         //fix for search by profile with address fields.
2396         $tName = "{$locationType[$locType[1]]}-address";
2397       }
2398       elseif (in_array($locType[0],
2399           [
2400             'on_hold',
2401             'signature_html',
2402             'signature_text',
2403             'is_bulkmail',
2404           ]
2405         )) {
2406         $tName = "{$locationType[$locType[1]]}-email";
2407       }
2408       elseif ($locType[0] == 'provider_id') {
2409         $tName = "{$locationType[$locType[1]]}-im";
2410       }
2411       elseif ($locType[0] == 'openid') {
2412         $tName = "{$locationType[$locType[1]]}-openid";
2413       }
2414       else {
2415         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2416       }
2417       $tName = str_replace(' ', '_', $tName);
2418       return [$tName, $fldName];
2419     }
2420     throw new CRM_Core_Exception('Cannot determine location table information');
2421   }
2422
2423   /**
2424    * Given a result dao, extract the values and return that array
2425    *
2426    * @param CRM_Core_DAO $dao
2427    *
2428    * @return array
2429    *   values for this query
2430    */
2431   public function store($dao) {
2432     $value = [];
2433
2434     foreach ($this->_element as $key => $dontCare) {
2435       if (property_exists($dao, $key)) {
2436         if (strpos($key, '-') !== FALSE) {
2437           $values = explode('-', $key);
2438           $lastElement = array_pop($values);
2439           $current = &$value;
2440           $cnt = count($values);
2441           $count = 1;
2442           foreach ($values as $v) {
2443             if (!array_key_exists($v, $current)) {
2444               $current[$v] = [];
2445             }
2446             //bad hack for im_provider
2447             if ($lastElement == 'provider_id') {
2448               if ($count < $cnt) {
2449                 $current = &$current[$v];
2450               }
2451               else {
2452                 $lastElement = "{$v}_{$lastElement}";
2453               }
2454             }
2455             else {
2456               $current = &$current[$v];
2457             }
2458             $count++;
2459           }
2460
2461           $current[$lastElement] = $dao->$key;
2462         }
2463         else {
2464           $value[$key] = $dao->$key;
2465         }
2466       }
2467     }
2468     return $value;
2469   }
2470
2471   /**
2472    * Getter for tables array.
2473    *
2474    * @return array
2475    */
2476   public function tables() {
2477     return $this->_tables;
2478   }
2479
2480   /**
2481    * Sometimes used to create the from clause, but, not reliably, set
2482    * this AND set tables.
2483    *
2484    * It's unclear the intent - there is a 'simpleFrom' clause which
2485    * takes whereTables into account & a fromClause which doesn't.
2486    *
2487    * logic may have eroded?
2488    *
2489    * @return array
2490    */
2491   public function whereTables() {
2492     return $this->_whereTables;
2493   }
2494
2495   /**
2496    * Generate the where clause (used in match contacts and permissions)
2497    *
2498    * @param array $params
2499    * @param array $fields
2500    * @param array $tables
2501    * @param $whereTables
2502    * @param bool $strict
2503    *
2504    * @return string
2505    * @throws \CRM_Core_Exception
2506    */
2507   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2508     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2509       FALSE, $strict
2510     );
2511
2512     $tables = array_merge($query->tables(), $tables);
2513     $whereTables = array_merge($query->whereTables(), $whereTables);
2514
2515     return $query->_whereClause;
2516   }
2517
2518   /**
2519    * Create the from clause.
2520    *
2521    * @param array $tables
2522    *   Tables that need to be included in this from clause. If null,
2523    *   return mimimal from clause (i.e. civicrm_contact).
2524    * @param array $inner
2525    *   Tables that should be inner-joined.
2526    * @param array $right
2527    *   Tables that should be right-joined.
2528    * @param bool $primaryLocation
2529    *   Search on primary location. See note below.
2530    * @param int $mode
2531    *   Determines search mode based on bitwise MODE_* constants.
2532    * @param string|NULL $apiEntity
2533    *   Determines search mode based on entity by string.
2534    *
2535    * The $primaryLocation flag only seems to be used when
2536    * locationType() has been called. This may be a search option
2537    * exposed, or perhaps it's a "search all details" approach which
2538    * predates decoupling of location types and primary fields?
2539    *
2540    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2541    *
2542    * @return string
2543    *   the from clause
2544    */
2545   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2546
2547     $from = ' FROM civicrm_contact contact_a';
2548     if (empty($tables)) {
2549       return $from;
2550     }
2551
2552     if (!empty($tables['civicrm_worldregion'])) {
2553       $tables = array_merge(['civicrm_country' => 1], $tables);
2554     }
2555
2556     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2557         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2558       $tables = array_merge(['civicrm_address' => 1],
2559         $tables
2560       );
2561     }
2562
2563     // add group_contact and group table is subscription history is present
2564     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2565       $tables = array_merge([
2566         'civicrm_group' => 1,
2567         'civicrm_group_contact' => 1,
2568       ],
2569         $tables
2570       );
2571     }
2572
2573     // to handle table dependencies of components
2574     CRM_Core_Component::tableNames($tables);
2575     // to handle table dependencies of hook injected tables
2576     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2577
2578     //format the table list according to the weight
2579     $info = CRM_Core_TableHierarchy::info();
2580
2581     foreach ($tables as $key => $value) {
2582       $k = 99;
2583       if (strpos($key, '-') !== FALSE) {
2584         $keyArray = explode('-', $key);
2585         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2586       }
2587       elseif (strpos($key, '_') !== FALSE) {
2588         $keyArray = explode('_', $key);
2589         if (is_numeric(array_pop($keyArray))) {
2590           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2591         }
2592         else {
2593           $k = CRM_Utils_Array::value($key, $info, 99);
2594         }
2595       }
2596       else {
2597         $k = CRM_Utils_Array::value($key, $info, 99);
2598       }
2599       $tempTable[$k . ".$key"] = $key;
2600     }
2601     ksort($tempTable);
2602     $newTables = [];
2603     foreach ($tempTable as $key) {
2604       $newTables[$key] = $tables[$key];
2605     }
2606
2607     $tables = $newTables;
2608
2609     foreach ($tables as $name => $value) {
2610       if (!$value) {
2611         continue;
2612       }
2613
2614       if (!empty($inner[$name])) {
2615         $side = 'INNER';
2616       }
2617       elseif (!empty($right[$name])) {
2618         $side = 'RIGHT';
2619       }
2620       else {
2621         $side = 'LEFT';
2622       }
2623
2624       if ($value != 1) {
2625         // if there is already a join statement in value, use value itself
2626         if (strpos($value, 'JOIN')) {
2627           $from .= " $value ";
2628         }
2629         else {
2630           $from .= " $side JOIN $name ON ( $value ) ";
2631         }
2632         continue;
2633       }
2634
2635       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2636     }
2637     return $from;
2638   }
2639
2640   /**
2641    * Get join statements for the from clause depending on entity type
2642    *
2643    * @param string $name
2644    * @param int $mode
2645    * @param string $side
2646    * @param string $primaryLocation
2647    * @return string
2648    */
2649   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2650     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2651     switch ($name) {
2652       case 'civicrm_address':
2653         //CRM-14263 further handling of address joins further down...
2654         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2655
2656       case 'civicrm_state_province':
2657         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2658         // but that will have been implied-in by the calling function.
2659         // test cover in testContactIDQuery
2660         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2661
2662       case 'civicrm_country':
2663         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2664         // but that will have been implied-in by the calling function.
2665         // test cover in testContactIDQuery
2666         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2667
2668       case 'civicrm_phone':
2669         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2670
2671       case 'civicrm_email':
2672         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2673
2674       case 'civicrm_im':
2675         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2676
2677       case 'im_provider':
2678         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2679         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2680         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2681         return $from;
2682
2683       case 'civicrm_openid':
2684         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2685
2686       case 'civicrm_worldregion':
2687         // We can be sure from the calling function that country will already be joined in.
2688         // we really don't need world_region - we could use a pseudoconstant for it.
2689         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2690
2691       case 'civicrm_location_type':
2692         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2693
2694       case 'civicrm_group':
2695         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2696
2697       case 'civicrm_group_contact':
2698         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2699
2700       case 'civicrm_group_contact_cache':
2701         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2702
2703       case 'civicrm_activity':
2704       case 'civicrm_activity_tag':
2705       case 'activity_type':
2706       case 'activity_status':
2707       case 'parent_id':
2708       case 'civicrm_activity_contact':
2709       case 'source_contact':
2710       case 'activity_priority':
2711         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2712
2713       case 'civicrm_entity_tag':
2714         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2715         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2716
2717       case 'civicrm_note':
2718         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2719         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2720
2721       case 'civicrm_subscription_history':
2722         $from = " $side JOIN civicrm_subscription_history";
2723         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2724         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2725
2726       case 'civicrm_relationship':
2727         if (self::$_relType == 'reciprocal') {
2728           if (self::$_relationshipTempTable) {
2729             // we have a temptable to join on
2730             $tbl = self::$_relationshipTempTable;
2731             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2732           }
2733           else {
2734             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2735             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2736             return $from;
2737           }
2738         }
2739         elseif (self::$_relType == 'b') {
2740           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2741           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2742         }
2743         else {
2744           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2745           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2746         }
2747
2748       case 'civicrm_log':
2749         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2750         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2751
2752       case 'civicrm_tag':
2753         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2754
2755       case 'civicrm_grant':
2756         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2757
2758       case 'civicrm_website':
2759         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2760
2761       case 'civicrm_campaign':
2762         //Move to default case if not in either mode.
2763         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2764           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2765         }
2766         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2767           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2768         }
2769         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2770           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2771         }
2772
2773       default:
2774         $locationTypeName = '';
2775         if (strpos($name, '-address') != 0) {
2776           $locationTypeName = 'address';
2777         }
2778         elseif (strpos($name, '-phone') != 0) {
2779           $locationTypeName = 'phone';
2780         }
2781         elseif (strpos($name, '-email') != 0) {
2782           $locationTypeName = 'email';
2783         }
2784         elseif (strpos($name, '-im') != 0) {
2785           $locationTypeName = 'im';
2786         }
2787         elseif (strpos($name, '-openid') != 0) {
2788           $locationTypeName = 'openid';
2789         }
2790
2791         if ($locationTypeName) {
2792           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2793           $parts = explode('-', $name);
2794           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2795           foreach ($locationTypes as $locationTypeID => $locationType) {
2796             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2797               $locationID = $locationTypeID;
2798             }
2799           }
2800           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2801         }
2802         else {
2803           $from = CRM_Core_Component::from($name, $mode, $side);
2804         }
2805         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2806
2807         return $from;
2808     }
2809   }
2810
2811   /**
2812    * WHERE / QILL clause for deleted_contacts
2813    *
2814    * @param array $values
2815    */
2816   public function deletedContacts($values) {
2817     list($_, $_, $value, $grouping, $_) = $values;
2818     if ($value) {
2819       // *prepend* to the relevant grouping as this is quite an important factor
2820       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2821     }
2822   }
2823
2824   /**
2825    * Where / qill clause for contact_type
2826    *
2827    * @param $values
2828    *
2829    * @throws \CRM_Core_Exception
2830    */
2831   public function contactType(&$values) {
2832     list($name, $op, $value, $grouping, $wildcard) = $values;
2833
2834     $subTypes = [];
2835     $clause = [];
2836
2837     // account for search builder mapping multiple values
2838     if (!is_array($value)) {
2839       $values = self::parseSearchBuilderString($value, 'String');
2840       if (is_array($values)) {
2841         $value = array_flip($values);
2842       }
2843     }
2844
2845     if (is_array($value)) {
2846       foreach ($value as $k => $v) {
2847         // fix for CRM-771
2848         if ($k) {
2849           $subType = NULL;
2850           $contactType = $k;
2851           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2852             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2853           }
2854
2855           if (!empty($subType)) {
2856             $subTypes[$subType] = 1;
2857           }
2858           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2859         }
2860       }
2861     }
2862     else {
2863       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2864       $contactType = $contactTypeANDSubType[0];
2865       $subType = $contactTypeANDSubType[1] ?? NULL;
2866       if (!empty($subType)) {
2867         $subTypes[$subType] = 1;
2868       }
2869       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2870     }
2871
2872     // fix for CRM-771
2873     if (!empty($clause)) {
2874       $quill = $clause;
2875       if ($op == 'IN' || $op == 'NOT IN') {
2876         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2877       }
2878       else {
2879         $type = array_pop($clause);
2880         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2881       }
2882
2883       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2884
2885       if (!empty($subTypes)) {
2886         $this->includeContactSubTypes($subTypes, $grouping);
2887       }
2888     }
2889   }
2890
2891   /**
2892    * Where / qill clause for contact_sub_type.
2893    *
2894    * @param array $values
2895    */
2896   public function contactSubType(&$values) {
2897     list($name, $op, $value, $grouping, $wildcard) = $values;
2898     $this->includeContactSubTypes($value, $grouping, $op);
2899   }
2900
2901   /**
2902    * @param $value
2903    * @param $grouping
2904    * @param string $op
2905    *
2906    * @throws \CRM_Core_Exception
2907    */
2908   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2909
2910     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2911       $op = key($value);
2912       $value = $value[$op];
2913     }
2914
2915     $clause = [];
2916     $alias = "contact_a.contact_sub_type";
2917     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2918
2919     $op = str_replace('IN', 'LIKE', $op);
2920     $op = str_replace('=', 'LIKE', $op);
2921     $op = str_replace('!', 'NOT ', $op);
2922
2923     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2924       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2925     }
2926     elseif (is_array($value)) {
2927       foreach ($value as $k => $v) {
2928         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2929       }
2930     }
2931     else {
2932       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2933     }
2934
2935     if (!empty($clause)) {
2936       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2937     }
2938     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2939   }
2940
2941   /**
2942    * Where / qill clause for groups.
2943    *
2944    * @param $values
2945    *
2946    * @throws \CRM_Core_Exception
2947    * @throws \Exception
2948    */
2949   public function group($values) {
2950     list($name, $op, $value, $grouping, $wildcard) = $values;
2951
2952     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2953     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2954       $op = key($value);
2955       $value = $value[$op];
2956     }
2957     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2958     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2959     // even one group might equate to multiple when looking at children so IN is simpler.
2960     // @todo - also look at != casting but there are rows below to review.
2961     $opReplacements = [
2962       'IS EMPTY' => 'IS NULL',
2963       'IS NOT EMPTY' => 'IS NOT NULL',
2964       '=' => 'IN',
2965     ];
2966     if (isset($opReplacements[$op])) {
2967       $op = $opReplacements[$op];
2968     }
2969
2970     if (strpos($op, 'NULL')) {
2971       $value = NULL;
2972     }
2973
2974     if (is_array($value) && count($value) > 1) {
2975       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2976         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2977       }
2978       $this->_useDistinct = TRUE;
2979     }
2980
2981     if (isset($value)) {
2982       $value = CRM_Utils_Array::value($op, $value, $value);
2983     }
2984
2985     if ($name === 'group_type') {
2986       $value = array_keys($this->getGroupsFromTypeCriteria($value));
2987     }
2988
2989     $regularGroupIDs = $smartGroupIDs = [];
2990     foreach ((array) $value as $id) {
2991       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
2992         $smartGroupIDs[] = (int) $id;
2993       }
2994       else {
2995         $regularGroupIDs[] = (int) trim($id);
2996       }
2997     }
2998     $hasNonSmartGroups = count($regularGroupIDs);
2999
3000     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3001
3002     $statusJoinClause = $this->getGroupStatusClause($grouping);
3003     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3004     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3005     $groupClause = [];
3006     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3007       // include child groups IDs if any
3008       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3009       foreach ($childGroupIds as $key => $id) {
3010         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3011           $smartGroupIDs[] = $id;
3012           unset($childGroupIds[$key]);
3013         }
3014       }
3015       if (count($childGroupIds)) {
3016         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3017       }
3018
3019       if (empty($regularGroupIDs)) {
3020         if ($isGroupStatusSearch) {
3021           $regularGroupIDs = $smartGroupIDs;
3022         }
3023         // If it is still empty we want a filter that blocks all results.
3024         if (empty($regularGroupIDs)) {
3025           $regularGroupIDs = [0];
3026         }
3027       }
3028
3029       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3030       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3031
3032       // @todo consider just casting != to NOT IN & handling both together.
3033       if ($op === '!=') {
3034         $groupIds = '';
3035         if (!empty($regularGroupIDs)) {
3036           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3037         }
3038         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3039       }
3040       else {
3041         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3042       }
3043       $groupClause[] = "( {$clause} )";
3044
3045       if ($statusJoinClause) {
3046         $joinClause[] = "{$gcTable}.$statusJoinClause";
3047       }
3048       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3049     }
3050
3051     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3052     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3053     // as that info is in the group_contact table.
3054     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3055       $this->_groupUniqueKey = uniqid();
3056       $this->_groupKeys[] = $this->_groupUniqueKey;
3057       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3058       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3059       if (!empty($groupContactCacheClause)) {
3060         if ($isNotOp) {
3061           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3062           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3063           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3064           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3065           if (strpos($op, 'IN') !== FALSE) {
3066             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3067           }
3068           else {
3069             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3070           }
3071         }
3072         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3073       }
3074     }
3075
3076     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3077     if (!empty($groupClause)) {
3078       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3079     }
3080
3081     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3082     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3083     if (strpos($op, 'NULL') === FALSE) {
3084       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3085     }
3086   }
3087
3088   /**
3089    * @return array
3090    */
3091   public function getGroupCacheTableKeys() {
3092     return $this->_groupKeys;
3093   }
3094
3095   /**
3096    * Function translates selection of group type into a list of groups.
3097    * @param $value
3098    *
3099    * @return array
3100    */
3101   public function getGroupsFromTypeCriteria($value) {
3102     $groupIds = [];
3103     foreach ((array) $value as $groupTypeValue) {
3104       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3105       $groupIds = ($groupIds + $groupList);
3106     }
3107     return $groupIds;
3108   }
3109
3110   /**
3111    * Prime smart group cache for smart groups in the search, and join
3112    * civicrm_group_contact_cache table into the query.
3113    *
3114    * @param array $groups IDs of groups specified in search criteria.
3115    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3116    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3117    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3118    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3119    *
3120    * @return string WHERE clause component for smart group criteria.
3121    * @throws \CRM_Core_Exception
3122    */
3123   public function addGroupContactCache($groups, $tableAlias, $joinTable = "contact_a", $op, $joinColumn = 'id') {
3124     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3125     $groupsIds = $groups;
3126
3127     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3128     if (!empty($operator[$op]) && is_array($groups)) {
3129       $op = $operator[$op];
3130     }
3131     if (!$isNullOp && !$groups) {
3132       return NULL;
3133     }
3134     elseif (strpos($op, 'IN') !== FALSE) {
3135       $groups = [$op => $groups];
3136     }
3137     elseif (is_array($groups) && count($groups)) {
3138       $groups = ['IN' => $groups];
3139     }
3140
3141     // Find all the groups that are part of a saved search.
3142     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3143     $sql = "
3144 SELECT id, cache_date, saved_search_id, children
3145 FROM   civicrm_group
3146 WHERE  $smartGroupClause
3147   AND  ( saved_search_id != 0
3148    OR    saved_search_id IS NOT NULL
3149    OR    children IS NOT NULL )
3150 ";
3151
3152     $group = CRM_Core_DAO::executeQuery($sql);
3153
3154     while ($group->fetch()) {
3155       $this->_useDistinct = TRUE;
3156       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3157         CRM_Contact_BAO_GroupContactCache::load($group);
3158       }
3159     }
3160     if ($group->N == 0 && $op != 'NOT IN') {
3161       return NULL;
3162     }
3163
3164     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3165
3166     if ($op == 'NOT IN') {
3167       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3168     }
3169     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3170   }
3171
3172   /**
3173    * Where / qill clause for cms users
3174    *
3175    * @param $values
3176    */
3177   public function ufUser(&$values) {
3178     list($name, $op, $value, $grouping, $wildcard) = $values;
3179
3180     if ($value == 1) {
3181       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3182
3183       $this->_qill[$grouping][] = ts('CMS User');
3184     }
3185     elseif ($value == 0) {
3186       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3187
3188       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3189       $this->_qill[$grouping][] = ts('Not a CMS User');
3190     }
3191   }
3192
3193   /**
3194    * All tag search specific.
3195    *
3196    * @param array $values
3197    *
3198    * @throws \CRM_Core_Exception
3199    */
3200   public function tagSearch(&$values) {
3201     list($name, $op, $value, $grouping, $wildcard) = $values;
3202
3203     $op = "LIKE";
3204     $value = "%{$value}%";
3205     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3206
3207     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3208     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3209
3210     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3211     $tTable = "`civicrm_tag-" . uniqid() . "`";
3212
3213     if ($useAllTagTypes[2]) {
3214       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3215         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3216             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3217
3218       // search tag in cases
3219       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3220       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3221       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3222         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3223             LEFT JOIN civicrm_case
3224             ON (civicrm_case_contact.case_id = civicrm_case.id
3225                 AND civicrm_case.is_deleted = 0 )
3226             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3227             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3228       // search tag in activities
3229       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3230       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3231       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3232       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3233
3234       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3235         = " LEFT JOIN civicrm_activity_contact
3236             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3237             LEFT JOIN civicrm_activity
3238             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3239             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3240             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3241             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3242
3243       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3244       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3245     }
3246     else {
3247       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3248       $tTable = "`civicrm_tag-" . uniqid() . "`";
3249       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3250       {$etTable}.entity_table = 'civicrm_contact' )
3251                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3252
3253       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3254       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3255     }
3256   }
3257
3258   /**
3259    * Where / qill clause for tag.
3260    *
3261    * @param array $values
3262    *
3263    * @throws \CRM_Core_Exception
3264    */
3265   public function tag(&$values) {
3266     list($name, $op, $value, $grouping, $wildcard) = $values;
3267
3268     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3269     // API/Search Builder format array(operator => array(values))
3270     if (is_array($value)) {
3271       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3272         $op = key($value);
3273         $value = $value[$op];
3274       }
3275       if (count($value) > 1) {
3276         $this->_useDistinct = TRUE;
3277       }
3278     }
3279
3280     // implode array, then remove all spaces
3281     $value = str_replace(' ', '', implode(',', (array) $value));
3282     if (!empty($value)) {
3283       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3284     }
3285
3286     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3287     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3288
3289     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3290
3291     if (!empty($useAllTagTypes[2])) {
3292       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3293         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3294
3295       // search tag in cases
3296       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3297       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3298       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3299
3300       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3301         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3302             LEFT JOIN civicrm_case
3303             ON (civicrm_case_contact.case_id = civicrm_case.id
3304                 AND civicrm_case.is_deleted = 0 )
3305             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3306       // search tag in activities
3307       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3308       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3309         = " LEFT JOIN civicrm_activity_contact
3310             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3311             LEFT JOIN civicrm_activity
3312             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3313             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3314             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3315
3316       // CRM-10338
3317       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3318         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3319       }
3320       else {
3321         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3322       }
3323     }
3324     else {
3325       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3326         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3327
3328       // CRM-10338
3329       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3330         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3331         $op = str_replace('EMPTY', 'NULL', $op);
3332         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3333       }
3334       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3335       elseif ($op == '!=') {
3336         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3337       }
3338       elseif ($op == '=' || strstr($op, 'IN')) {
3339         $op = ($op == '=') ? 'IN' : $op;
3340         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3341       }
3342     }
3343     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3344   }
3345
3346   /**
3347    * Where/qill clause for notes
3348    *
3349    * @param array $values
3350    *
3351    * @throws \CRM_Core_Exception
3352    */
3353   public function notes(&$values) {
3354     list($name, $op, $value, $grouping, $wildcard) = $values;
3355
3356     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3357     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3358     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3359
3360     $this->_useDistinct = TRUE;
3361
3362     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3363       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3364
3365     $n = trim($value);
3366     $value = CRM_Core_DAO::escapeString($n);
3367     if ($wildcard) {
3368       if (strpos($value, '%') === FALSE) {
3369         $value = "%$value%";
3370       }
3371       $op = 'LIKE';
3372     }
3373     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3374       $value = NULL;
3375     }
3376
3377     $label = NULL;
3378     $clauses = [];
3379     if ($noteOption % 2 == 0) {
3380       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3381       $label = ts('Note: Body Only');
3382     }
3383     if ($noteOption % 3 == 0) {
3384       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3385       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3386     }
3387     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3388     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3389     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3390   }
3391
3392   /**
3393    * @param string $name
3394    * @param $op
3395    * @param $grouping
3396    *
3397    * @return bool
3398    */
3399   public function nameNullOrEmptyOp($name, $op, $grouping) {
3400     switch ($op) {
3401       case 'IS NULL':
3402       case 'IS NOT NULL':
3403         $this->_where[$grouping][] = "contact_a.$name $op";
3404         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3405         return TRUE;
3406
3407       case 'IS EMPTY':
3408         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3409         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3410         return TRUE;
3411
3412       case 'IS NOT EMPTY':
3413         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3414         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3415         return TRUE;
3416
3417       default:
3418         return FALSE;
3419     }
3420   }
3421
3422   /**
3423    * Where / qill clause for sort_name
3424    *
3425    * @param array $values
3426    */
3427   public function sortName(&$values) {
3428     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3429
3430     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3431     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3432       return;
3433     }
3434
3435     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3436
3437     if (!strlen($value)) {
3438       return;
3439     }
3440
3441     $config = CRM_Core_Config::singleton();
3442
3443     $sub = [];
3444
3445     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3446     $subGlue = ' OR ';
3447
3448     $firstChar = substr($value, 0, 1);
3449     $lastChar = substr($value, -1, 1);
3450     $quotes = ["'", '"'];
3451     // If string is quoted, strip quotes and otherwise don't alter it
3452     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3453       $value = trim($value, implode('', $quotes));
3454     }
3455     // Replace spaces with wildcards for a LIKE operation
3456     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3457     // Also need to check if there is space in between sort name.
3458     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') === TRUE) {
3459       $value = str_replace(' ', '%', $value);
3460     }
3461     $value = CRM_Core_DAO::escapeString(trim($value));
3462     if (strlen($value)) {
3463       $fieldsub = [];
3464       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3465       if ($fieldName == 'sort_name') {
3466         $wc = "contact_a.sort_name";
3467       }
3468       else {
3469         $wc = "contact_a.display_name";
3470       }
3471       $fieldsub[] = " ( $wc $op $value )";
3472       if ($config->includeNickNameInName) {
3473         $wc = "contact_a.nick_name";
3474         $fieldsub[] = " ( $wc $op $value )";
3475       }
3476       if ($config->includeEmailInName) {
3477         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3478       }
3479       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3480     }
3481
3482     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3483
3484     $this->_where[$grouping][] = $sub;
3485     if ($config->includeEmailInName) {
3486       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3487       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3488     }
3489     else {
3490       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3491     }
3492   }
3493
3494   /**
3495    * Where/qill clause for greeting fields.
3496    *
3497    * @param array $values
3498    *
3499    * @throws \CRM_Core_Exception
3500    */
3501   public function greetings(&$values) {
3502     list($name, $op, $value, $grouping, $wildcard) = $values;
3503     $name .= '_display';
3504
3505     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3506     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3507     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3508   }
3509
3510   /**
3511    * Where / qill clause for email
3512    *
3513    * @param array $values
3514    * @param string $isForcePrimaryOnly
3515    *
3516    * @throws \CRM_Core_Exception
3517    */
3518   protected function email(&$values, $isForcePrimaryOnly) {
3519     list($name, $op, $value, $grouping, $wildcard) = $values;
3520     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3521
3522     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3523     // so following code is use build whereClause for contact's primart email id
3524     if (!empty($isForcePrimaryOnly)) {
3525       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3526     }
3527     // @todo - this should come from the $this->_fields array
3528     $dbName = $name === 'email_id' ? 'id' : $name;
3529
3530     if (is_array($value) || $name === 'email_id') {
3531       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3532       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3533       return;
3534     }
3535
3536     // Is this ever hit now? Ideally ensure always an array & handle above.
3537     $n = trim($value);
3538     if ($n) {
3539       if (substr($n, 0, 1) == '"' &&
3540         substr($n, -1, 1) == '"'
3541       ) {
3542         $n = substr($n, 1, -1);
3543         $value = CRM_Core_DAO::escapeString($n);
3544         $op = '=';
3545       }
3546       else {
3547         $value = self::getWildCardedValue($wildcard, $op, $n);
3548       }
3549       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3550       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3551     }
3552     else {
3553       $this->_qill[$grouping][] = ts('Email') . " $op ";
3554       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3555     }
3556   }
3557
3558   /**
3559    * Where / qill clause for phone number
3560    *
3561    * @param array $values
3562    *
3563    * @throws \CRM_Core_Exception
3564    */
3565   public function phone_numeric(&$values) {
3566     list($name, $op, $value, $grouping, $wildcard) = $values;
3567     // Strip non-numeric characters; allow wildcards
3568     $number = preg_replace('/[^\d%]/', '', $value);
3569     if ($number) {
3570       if (strpos($number, '%') === FALSE) {
3571         $number = "%$number%";
3572       }
3573
3574       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3575       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3576       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3577     }
3578   }
3579
3580   /**
3581    * Where / qill clause for phone type/location
3582    *
3583    * @param array $values
3584    *
3585    * @throws \CRM_Core_Exception
3586    */
3587   public function phone_option_group($values) {
3588     list($name, $op, $value, $grouping, $wildcard) = $values;
3589     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3590     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3591     $optionName = $options[$value];
3592     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3593     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3594     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3595   }
3596
3597   /**
3598    * Where / qill clause for street_address.
3599    *
3600    * @param array $values
3601    *
3602    * @throws \CRM_Core_Exception
3603    */
3604   public function street_address(&$values) {
3605     list($name, $op, $value, $grouping) = $values;
3606
3607     if (!$op) {
3608       $op = 'LIKE';
3609     }
3610
3611     $n = trim($value);
3612
3613     if ($n) {
3614       if (strpos($value, '%') === FALSE) {
3615         // only add wild card if not there
3616         $value = "%{$value}%";
3617       }
3618       $op = 'LIKE';
3619       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3620       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3621     }
3622     else {
3623       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3624       $this->_qill[$grouping][] = ts('Street') . " $op ";
3625     }
3626
3627     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3628   }
3629
3630   /**
3631    * Where / qill clause for street_unit.
3632    *
3633    * @param array $values
3634    *
3635    * @throws \CRM_Core_Exception
3636    */
3637   public function street_number(&$values) {
3638     list($name, $op, $value, $grouping, $wildcard) = $values;
3639
3640     if (!$op) {
3641       $op = '=';
3642     }
3643
3644     $n = trim($value);
3645
3646     if (strtolower($n) == 'odd') {
3647       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3648       $this->_qill[$grouping][] = ts('Street Number is odd');
3649     }
3650     elseif (strtolower($n) == 'even') {
3651       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3652       $this->_qill[$grouping][] = ts('Street Number is even');
3653     }
3654     else {
3655       $value = $n;
3656       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3657       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3658     }
3659
3660     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3661   }
3662
3663   /**
3664    * Where / qill clause for sorting by character.
3665    *
3666    * @param array $values
3667    */
3668   public function sortByCharacter(&$values) {
3669     list($name, $op, $value, $grouping, $wildcard) = $values;
3670
3671     $name = trim($value);
3672     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3673     $this->_where[$grouping][] = $cond;
3674     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3675   }
3676
3677   /**
3678    * Where / qill clause for including contact ids.
3679    */
3680   public function includeContactIDs() {
3681     if (!$this->_includeContactIds || empty($this->_params)) {
3682       return;
3683     }
3684
3685     $contactIds = [];
3686     foreach ($this->_params as $id => $values) {
3687       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3688         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3689       }
3690     }
3691     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3692     if (!empty($contactIds)) {
3693       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3694     }
3695   }
3696
3697   /**
3698    * Where / qill clause for postal code.
3699    *
3700    * @param array $values
3701    *
3702    * @throws \CRM_Core_Exception
3703    */
3704   public function postalCode(&$values) {
3705     // skip if the fields dont have anything to do with postal_code
3706     if (empty($this->_fields['postal_code'])) {
3707       return;
3708     }
3709
3710     list($name, $op, $value, $grouping, $wildcard) = $values;
3711
3712     // Handle numeric postal code range searches properly by casting the column as numeric
3713     if (is_numeric($value)) {
3714       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3715       $val = CRM_Utils_Type::escape($value, 'Integer');
3716     }
3717     else {
3718       $field = 'civicrm_address.postal_code';
3719       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3720       if (!is_array($value)) {
3721         $val = CRM_Utils_Type::escape($value, 'String');
3722       }
3723       else {
3724         // Do we need to escape values here? I would expect buildClause does.
3725         $val = $value;
3726       }
3727     }
3728
3729     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3730
3731     if ($name == 'postal_code') {
3732       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3733       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3734     }
3735     elseif ($name == 'postal_code_low') {
3736       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3737       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3738     }
3739     elseif ($name == 'postal_code_high') {
3740       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3741       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3742     }
3743   }
3744
3745   /**
3746    * Where / qill clause for location type.
3747    *
3748    * @param array $values
3749    * @param null $status
3750    *
3751    * @return string
3752    */
3753   public function locationType(&$values, $status = NULL) {
3754     list($name, $op, $value, $grouping, $wildcard) = $values;
3755
3756     if (is_array($value)) {
3757       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3758       $this->_tables['civicrm_address'] = 1;
3759       $this->_whereTables['civicrm_address'] = 1;
3760
3761       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3762       $names = [];
3763       foreach ($value as $id) {
3764         $names[] = $locationType[$id];
3765       }
3766
3767       $this->_primaryLocation = FALSE;
3768
3769       if (!$status) {
3770         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3771       }
3772       else {
3773         return implode(' ' . ts('or') . ' ', $names);
3774       }
3775     }
3776   }
3777
3778   /**
3779    * @param $values
3780    * @param bool $fromStateProvince
3781    *
3782    * @return array|NULL
3783    * @throws \CRM_Core_Exception
3784    */
3785   public function country(&$values, $fromStateProvince = TRUE) {
3786     list($name, $op, $value, $grouping, $wildcard) = $values;
3787
3788     if (!$fromStateProvince) {
3789       $stateValues = $this->getWhereValues('state_province', $grouping);
3790       if (!empty($stateValues)) {
3791         // return back to caller if there are state province values
3792         // since that handles this case
3793         return NULL;
3794       }
3795     }
3796
3797     $countryClause = $countryQill = NULL;
3798     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3799       $this->_tables['civicrm_address'] = 1;
3800       $this->_whereTables['civicrm_address'] = 1;
3801
3802       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3803       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3804       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3805
3806       if (!$fromStateProvince) {
3807         $this->_where[$grouping][] = $countryClause;
3808         $this->_qill[$grouping][] = $countryQill;
3809       }
3810     }
3811
3812     if ($fromStateProvince) {
3813       if (!empty($countryClause)) {
3814         return [
3815           $countryClause,
3816           " ...AND... " . $countryQill,
3817         ];
3818       }
3819       else {
3820         return [NULL, NULL];
3821       }
3822     }
3823   }
3824
3825   /**
3826    * Where / qill clause for county (if present).
3827    *
3828    * @param array $values
3829    * @param null $status
3830    *
3831    * @return string
3832    */
3833   public function county(&$values, $status = NULL) {
3834     list($name, $op, $value, $grouping, $wildcard) = $values;
3835
3836     if (!is_array($value)) {
3837       // force the county to be an array
3838       $value = [$value];
3839     }
3840
3841     // check if the values are ids OR names of the counties
3842     $inputFormat = 'id';
3843     foreach ($value as $v) {
3844       if (!is_numeric($v)) {
3845         $inputFormat = 'name';
3846         break;
3847       }
3848     }
3849     $names = [];
3850     if ($op == '=') {
3851       $op = 'IN';
3852     }
3853     elseif ($op == '!=') {
3854       $op = 'NOT IN';
3855     }
3856     else {
3857       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3858       $op = str_replace('EMPTY', 'NULL', $op);
3859     }
3860
3861     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3862       $clause = "civicrm_address.county_id $op";
3863     }
3864     elseif ($inputFormat == 'id') {
3865       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3866
3867       $county = CRM_Core_PseudoConstant::county();
3868       foreach ($value as $id) {
3869         $names[] = $county[$id] ?? NULL;
3870       }
3871     }
3872     else {
3873       $inputClause = [];
3874       $county = CRM_Core_PseudoConstant::county();
3875       foreach ($value as $name) {
3876         $name = trim($name);
3877         $inputClause[] = CRM_Utils_Array::key($name, $county);
3878       }
3879       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3880       $names = $value;
3881     }
3882     $this->_tables['civicrm_address'] = 1;
3883     $this->_whereTables['civicrm_address'] = 1;
3884
3885     $this->_where[$grouping][] = $clause;
3886     if (!$status) {
3887       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3888     }
3889     else {
3890       return implode(' ' . ts('or') . ' ', $names);
3891     }
3892   }
3893
3894   /**
3895    * Where / qill clause for state/province AND country (if present).
3896    *
3897    * @param array $values
3898    * @param null $status
3899    *
3900    * @return string
3901    * @throws \CRM_Core_Exception
3902    */
3903   public function stateProvince(&$values, $status = NULL) {
3904     list($name, $op, $value, $grouping, $wildcard) = $values;
3905
3906     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3907     $this->_tables['civicrm_address'] = 1;
3908     $this->_whereTables['civicrm_address'] = 1;
3909
3910     $countryValues = $this->getWhereValues('country', $grouping);
3911     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3912     if ($countryClause) {
3913       $clause = "( $stateClause AND $countryClause )";
3914     }
3915     else {
3916       $clause = $stateClause;
3917     }
3918
3919     $this->_where[$grouping][] = $clause;
3920     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3921     if (!$status) {
3922       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3923     }
3924     else {
3925       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3926     }
3927   }
3928
3929   /**
3930    * Where / qill clause for change log.
3931    *
3932    * @param array $values
3933    */
3934   public function changeLog(&$values) {
3935     list($name, $op, $value, $grouping, $wildcard) = $values;
3936
3937     $targetName = $this->getWhereValues('changed_by', $grouping);
3938     if (!$targetName) {
3939       return;
3940     }
3941
3942     $name = trim($targetName[2]);
3943     $name = CRM_Core_DAO::escapeString($name);
3944     $name = $targetName[4] ? "%$name%" : $name;
3945     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3946     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3947     $fieldTitle = ts('Altered By');
3948
3949     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3950     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3951       1 => $fieldTitle,
3952       2 => $qillop,
3953       3 => $qillVal,
3954     ]);
3955   }
3956
3957   /**
3958    * @param $values
3959    *
3960    * @throws \CRM_Core_Exception
3961    */
3962   public function modifiedDates($values) {
3963     $this->_useDistinct = TRUE;
3964
3965     // CRM-11281, default to added date if not set
3966     $fieldTitle = ts('Added Date');
3967     $fieldName = 'created_date';
3968     foreach (array_keys($this->_params) as $id) {
3969       if ($this->_params[$id][0] == 'log_date') {
3970         if ($this->_params[$id][2] == 2) {
3971           $fieldTitle = ts('Modified Date');
3972           $fieldName = 'modified_date';
3973         }
3974       }
3975     }
3976
3977     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
3978
3979     self::$_openedPanes[ts('Change Log')] = TRUE;
3980   }
3981
3982   /**
3983    * @param $values
3984    *
3985    * @throws \CRM_Core_Exception
3986    */
3987   public function demographics(&$values) {
3988     list($name, $op, $value, $grouping, $wildcard) = $values;
3989
3990     if (($name == 'age_low') || ($name == 'age_high')) {
3991       $this->ageRangeQueryBuilder($values,
3992         'contact_a', 'age', 'birth_date', ts('Age')
3993       );
3994     }
3995     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
3996
3997       $this->dateQueryBuilder($values,
3998         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
3999       );
4000     }
4001     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4002
4003       $this->dateQueryBuilder($values,
4004         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4005       );
4006     }
4007
4008     self::$_openedPanes[ts('Demographics')] = TRUE;
4009   }
4010
4011   /**
4012    * @param $values
4013    */
4014   public function privacy(&$values) {
4015     list($name, $op, $value, $grouping) = $values;
4016     if (is_array($value)) {
4017       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4018         $op = key($value);
4019         $value = $value[$op];
4020       }
4021     }
4022     $field = $this->_fields[$name] ?? NULL;
4023     CRM_Utils_Type::validate($value, 'Integer');
4024     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4025     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4026     $title = $field ? $field['title'] : $name;
4027     $this->_qill[$grouping][] = "$title $op $value";
4028   }
4029
4030   /**
4031    * @param $values
4032    */
4033   public function privacyOptions($values) {
4034     list($name, $op, $value, $grouping, $wildcard) = $values;
4035
4036     if (empty($value) || !is_array($value)) {
4037       return;
4038     }
4039
4040     // get the operator and toggle values
4041     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4042     $operator = 'OR';
4043     if ($opValues &&
4044       strtolower($opValues[2] == 'AND')
4045     ) {
4046       // @todo this line is logially unreachable
4047       $operator = 'AND';
4048     }
4049
4050     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4051     $compareOP = '!';
4052     if ($toggleValues &&
4053       $toggleValues[2] == 2
4054     ) {
4055       $compareOP = '';
4056     }
4057
4058     $clauses = [];
4059     $qill = [];
4060     foreach ($value as $dontCare => $pOption) {
4061       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4062       $field = $this->_fields[$pOption] ?? NULL;
4063       $title = $field ? $field['title'] : $pOption;
4064       $qill[] = " $title = 1 ";
4065     }
4066
4067     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4068     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4069   }
4070
4071   /**
4072    * @param $values
4073    *
4074    * @throws \CRM_Core_Exception
4075    */
4076   public function preferredCommunication(&$values) {
4077     list($name, $op, $value, $grouping, $wildcard) = $values;
4078
4079     if (!is_array($value)) {
4080       $value = str_replace(['(', ')'], '', explode(",", $value));
4081     }
4082     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4083       $op = key($value);
4084       $value = $value[$op];
4085     }
4086     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4087
4088     if (self::caseImportant($op)) {
4089       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4090       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4091       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4092     }
4093
4094     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4095     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4096   }
4097
4098   /**
4099    * Where / qill clause for relationship.
4100    *
4101    * @param array $values
4102    */
4103   public function relationship(&$values) {
4104     list($name, $op, $value, $grouping, $wildcard) = $values;
4105     if ($this->_relationshipValuesAdded) {
4106       return;
4107     }
4108     // also get values array for relation_target_name
4109     // for relationship search we always do wildcard
4110     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4111     $description = $this->getWhereValues('relation_description', $grouping);
4112     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4113     $relStatus = $this->getWhereValues('relation_status', $grouping);
4114     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4115
4116     $nameClause = $name = NULL;
4117     if ($targetName) {
4118       $name = trim($targetName[2]);
4119       if (substr($name, 0, 1) == '"' &&
4120         substr($name, -1, 1) == '"'
4121       ) {
4122         $name = substr($name, 1, -1);
4123         $name = CRM_Core_DAO::escapeString($name);
4124         $nameClause = "= '$name'";
4125       }
4126       else {
4127         $name = CRM_Core_DAO::escapeString($name);
4128         $nameClause = "LIKE '%{$name}%'";
4129       }
4130     }
4131
4132     $relTypes = $relTypesIds = [];
4133     if (!empty($relationType)) {
4134       $relationType[2] = (array) $relationType[2];
4135       foreach ($relationType[2] as $relType) {
4136         $rel = explode('_', $relType);
4137         self::$_relType = $rel[1];
4138         $params = ['id' => $rel[0]];
4139         $typeValues = [];
4140         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4141         if (!empty($rTypeValue)) {
4142           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4143             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4144             self::$_relType = 'reciprocal';
4145           }
4146           $relTypesIds[] = $rel[0];
4147           $relTypes[] = $relType;
4148         }
4149       }
4150     }
4151
4152     // if we are creating a temp table we build our own where for the relationship table
4153     $relationshipTempTable = NULL;
4154     if (self::$_relType == 'reciprocal') {
4155       $where = [];
4156       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4157         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4158         ->getName();
4159       if ($nameClause) {
4160         $where[$grouping][] = " sort_name $nameClause ";
4161       }
4162       $groupJoinTable = "civicrm_relationship";
4163       $groupJoinColumn = "contact_id_alt";
4164     }
4165     else {
4166       $where = &$this->_where;
4167       if ($nameClause) {
4168         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4169       }
4170       $groupJoinTable = "contact_b";
4171       $groupJoinColumn = "id";
4172     }
4173     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4174     if ($nameClause || !$targetGroup) {
4175       if (!empty($relationType)) {
4176         $relQill = '';
4177         foreach ($relTypes as $rel) {
4178           if (!empty($relQill)) {
4179             $relQill .= ' OR ';
4180           }
4181           $relQill .= $allRelationshipType[$rel];
4182         }
4183         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4184       }
4185       elseif ($name) {
4186         $this->_qill[$grouping][] = $name;
4187       }
4188     }
4189
4190     //check to see if the target contact is in specified group
4191     if ($targetGroup) {
4192       //add contacts from static groups
4193       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4194         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4195       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4196         implode(",", $targetGroup[2]) . ") ) ";
4197
4198       //add contacts from saved searches
4199       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4200
4201       //set the group where clause
4202       if ($ssWhere) {
4203         $groupWhere[] = "( " . $ssWhere . " )";
4204       }
4205       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4206
4207       //Get the names of the target groups for the qill
4208       $groupNames = CRM_Core_PseudoConstant::group();
4209       $qillNames = [];
4210       foreach ($targetGroup[2] as $groupId) {
4211         if (array_key_exists($groupId, $groupNames)) {
4212           $qillNames[] = $groupNames[$groupId];
4213         }
4214       }
4215       if (!empty($relationType)) {
4216         $relQill = '';
4217         foreach ($relTypes as $rel) {
4218           if (!empty($relQill)) {
4219             $relQill .= ' OR ';
4220           }
4221           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4222         }
4223         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4224       }
4225       else {
4226         $this->_qill[$grouping][] = implode(", ", $qillNames);
4227       }
4228     }
4229
4230     // Description
4231     if (!empty($description[2]) && trim($description[2])) {
4232       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4233       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4234       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4235     }
4236
4237     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4238     $today = date('Ymd');
4239     //check for active, inactive and all relation status
4240     if (empty($relStatus[2])) {
4241       $where[$grouping][] = "(
4242 civicrm_relationship.is_active = 1 AND
4243 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4244 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4245 )";
4246       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4247     }
4248     elseif ($relStatus[2] == 1) {
4249       $where[$grouping][] = "(
4250 civicrm_relationship.is_active = 0 OR
4251 civicrm_relationship.end_date < {$today} OR
4252 civicrm_relationship.start_date > {$today}
4253 )";
4254       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4255     }
4256
4257     $onlyDeleted = 0;
4258     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4259       $onlyDeleted = 1;
4260     }
4261     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4262
4263     $this->addRelationshipPermissionClauses($grouping, $where);
4264     $this->addRelationshipDateClauses($grouping, $where);
4265     $this->addRelationshipActivePeriodClauses($grouping, $where);
4266     if (!empty($relTypes)) {
4267       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4268     }
4269     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4270     $this->_useDistinct = TRUE;
4271     $this->_relationshipValuesAdded = TRUE;
4272     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4273     // join on that,
4274     if ($relationshipTempTable) {
4275       $whereClause = '';
4276       if (!empty($where[$grouping])) {
4277         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4278         $whereClause = str_replace('contact_b', 'c', $whereClause);
4279       }
4280       $sql = "
4281         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4282           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4283             FROM civicrm_relationship
4284             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4285             $whereClause )
4286           UNION
4287             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4288             FROM civicrm_relationship
4289             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4290             $whereClause )
4291       ";
4292       CRM_Core_DAO::executeQuery($sql);
4293     }
4294   }
4295
4296   /**
4297    * Add relationship permission criteria to where clause.
4298    *
4299    * @param string $grouping
4300    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4301    *   Not the main query.
4302    */
4303   public function addRelationshipPermissionClauses($grouping, &$where) {
4304     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4305     if ($relPermission) {
4306       if (!is_array($relPermission[2])) {
4307         // this form value was scalar in previous versions of Civi
4308         $relPermission[2] = [$relPermission[2]];
4309       }
4310       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4311
4312       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4313
4314       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4315       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4316     }
4317   }
4318
4319   /**
4320    * Add start & end date criteria in
4321    * @param string $grouping
4322    * @param array $where
4323    *   = array to add where clauses to, in case you are generating a temp table.
4324    * not the main query.
4325    */
4326   public function addRelationshipDateClauses($grouping, &$where) {
4327     foreach (['start_date', 'end_date'] as $dateField) {
4328       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4329       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4330       if (!empty($dateValueLow)) {
4331         $date = date('Ymd', strtotime($dateValueLow[2]));
4332         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4333         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4334       }
4335       if (!empty($dateValueHigh)) {
4336         $date = date('Ymd', strtotime($dateValueHigh[2]));
4337         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4338         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4339       }
4340     }
4341   }
4342
4343   /**
4344    * Add start & end active period criteria in
4345    * @param string $grouping
4346    * @param array $where
4347    *   = array to add where clauses to, in case you are generating a temp table.
4348    * not the main query.
4349    */
4350   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4351     $dateValues = [];
4352     $dateField = 'active_period_date';
4353
4354     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4355     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4356     $dateValueLowFormated = $dateValueHighFormated = NULL;
4357     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4358       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4359       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4360       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4361     }
4362     elseif (!empty($dateValueLow)) {
4363       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4364       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4365     }
4366     elseif (!empty($dateValueHigh)) {
4367       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4368       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4369     }
4370
4371     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4372       $where[$grouping][] = $activePeriodClauses;
4373     }
4374   }
4375
4376   /**
4377    * Get start & end active period criteria
4378    *
4379    * @param $from
4380    * @param $to
4381    * @param $forceTableName
4382    *
4383    * @return string
4384    */
4385   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4386     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4387     if (!is_null($from) && !is_null($to)) {
4388       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4389                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4390                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4391                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4392                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4393                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4394     }
4395     elseif (!is_null($from)) {
4396       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4397                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4398                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4399     }
4400     elseif (!is_null($to)) {
4401       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4402                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4403                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4404     }
4405   }
4406
4407   /**
4408    * Default set of return properties.
4409    *
4410    * @param int $mode
4411    *
4412    * @return array
4413    *   derault return properties
4414    */
4415   public static function &defaultReturnProperties($mode = 1) {
4416     if (!isset(self::$_defaultReturnProperties)) {
4417       self::$_defaultReturnProperties = [];
4418     }
4419
4420     if (!isset(self::$_defaultReturnProperties[$mode])) {
4421       // add activity return properties
4422       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4423         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4424       }
4425       else {
4426         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4427       }
4428
4429       if (empty(self::$_defaultReturnProperties[$mode])) {
4430         self::$_defaultReturnProperties[$mode] = [
4431           'home_URL' => 1,
4432           'image_URL' => 1,
4433           'legal_identifier' => 1,
4434           'external_identifier' => 1,
4435           'contact_type' => 1,
4436           'contact_sub_type' => 1,
4437           'sort_name' => 1,
4438           'display_name' => 1,
4439           'preferred_mail_format' => 1,
4440           'nick_name' => 1,
4441           'first_name' => 1,
4442           'middle_name' => 1,
4443           'last_name' => 1,
4444           'prefix_id' => 1,
4445           'suffix_id' => 1,
4446           'formal_title' => 1,
4447           'communication_style_id' => 1,
4448           'birth_date' => 1,
4449           'gender_id' => 1,
4450           'street_address' => 1,
4451           'supplemental_address_1' => 1,
4452           'supplemental_address_2' => 1,
4453           'supplemental_address_3' => 1,
4454           'city' => 1,
4455           'postal_code' => 1,
4456           'postal_code_suffix' => 1,
4457           'state_province' => 1,
4458           'country' => 1,
4459           'world_region' => 1,
4460           'geo_code_1' => 1,
4461           'geo_code_2' => 1,
4462           'email' => 1,
4463           'on_hold' => 1,
4464           'phone' => 1,
4465           'im' => 1,
4466           'household_name' => 1,
4467           'organization_name' => 1,
4468           'deceased_date' => 1,
4469           'is_deceased' => 1,
4470           'job_title' => 1,
4471           'legal_name' => 1,
4472           'sic_code' => 1,
4473           'current_employer' => 1,
4474           // FIXME: should we use defaultHierReturnProperties() for the below?
4475           'do_not_email' => 1,
4476           'do_not_mail' => 1,
4477           'do_not_sms' => 1,
4478           'do_not_phone' => 1,
4479           'do_not_trade' => 1,
4480           'is_opt_out' => 1,
4481           'contact_is_deleted' => 1,
4482           'preferred_communication_method' => 1,
4483           'preferred_language' => 1,
4484         ];
4485       }
4486     }
4487     return self::$_defaultReturnProperties[$mode];
4488   }
4489
4490   /**
4491    * Get primary condition for a sql clause.
4492    *
4493    * @param int $value
4494    *
4495    * @return string|NULL
4496    */
4497   public static function getPrimaryCondition($value) {
4498     if (is_numeric($value)) {
4499       $value = (int ) $value;
4500       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4501     }
4502     return NULL;
4503   }
4504
4505   /**
4506    * Wrapper for a simple search query.
4507    *
4508    * @param array $params
4509    * @param array $returnProperties
4510    * @param bool $count
4511    *
4512    * @return string
4513    * @throws \CRM_Core_Exception
4514    */
4515   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4516     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4517     list($select, $from, $where, $having) = $query->query();
4518     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4519
4520     $query = "$select $from $where $groupBy $having";
4521     return $query;
4522   }
4523
4524   /**
4525    * These are stub comments as this function needs more explanation - particularly in terms of how it
4526    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4527    *
4528    * This function was originally written as a wrapper for the api query but is called from multiple places
4529    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4530    * but it is unclear as to whehter that is historical or there is a reason
4531    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4532    *
4533    * @param array $params
4534    * @param array $returnProperties
4535    * @param null $fields
4536    * @param string $sort
4537    * @param int $offset
4538    * @param int $row_count
4539    * @param bool $smartGroupCache
4540    *   ?? update smart group cache?.
4541    * @param bool $count
4542    *   Return count obnly.
4543    * @param bool $skipPermissions
4544    *   Should permissions be ignored or should the logged in user's permissions be applied.
4545    * @param int $mode
4546    *   This basically correlates to the component.
4547    * @param string $apiEntity
4548    *   The api entity being called.
4549    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4550    *
4551    * @param bool|null $primaryLocationOnly
4552    *
4553    * @return array
4554    * @throws \CRM_Core_Exception
4555    */
4556   public static function apiQuery(
4557     $params = NULL,
4558     $returnProperties = NULL,
4559     $fields = NULL,
4560     $sort = NULL,
4561     $offset = 0,
4562     $row_count = 25,
4563     $smartGroupCache = TRUE,
4564     $count = FALSE,
4565     $skipPermissions = TRUE,
4566     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4567     $apiEntity = NULL,
4568     $primaryLocationOnly = NULL
4569   ) {
4570
4571     $query = new CRM_Contact_BAO_Query(
4572       $params, $returnProperties,
4573       NULL, TRUE, FALSE, $mode,
4574       $skipPermissions,
4575       TRUE, $smartGroupCache,
4576       NULL, 'AND',
4577       $apiEntity, $primaryLocationOnly
4578     );
4579
4580     //this should add a check for view deleted if permissions are enabled
4581     if ($skipPermissions) {
4582       $query->_skipDeleteClause = TRUE;
4583     }
4584     $query->generatePermissionClause(FALSE, $count);
4585
4586     // note : this modifies _fromClause and _simpleFromClause
4587     $query->includePseudoFieldsJoin($sort);
4588
4589     list($select, $from, $where, $having) = $query->query($count);
4590
4591     $options = $query->_options;
4592     if (!empty($query->_permissionWhereClause)) {
4593       if (empty($where)) {
4594         $where = "WHERE $query->_permissionWhereClause";
4595       }
4596       else {
4597         $where = "$where AND $query->_permissionWhereClause";
4598       }
4599     }
4600
4601     $sql = "$select $from $where $having";
4602
4603     // add group by only when API action is not getcount
4604     //  otherwise query fetches incorrect count
4605     if ($query->_useGroupBy && !$count) {
4606       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4607     }
4608     if (!empty($sort)) {
4609       $sort = CRM_Utils_Type::escape($sort, 'String');
4610       $sql .= " ORDER BY $sort ";
4611     }
4612     if ($row_count > 0 && $offset >= 0) {
4613       $offset = CRM_Utils_Type::escape($offset, 'Int');
4614       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4615       $sql .= " LIMIT $offset, $row_count ";
4616     }
4617
4618     $dao = CRM_Core_DAO::executeQuery($sql);
4619
4620     // @todo derive this from the component class rather than hard-code two options.
4621     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4622
4623     $values = [];
4624     while ($dao->fetch()) {
4625       if ($count) {
4626         $noRows = $dao->rowCount;
4627         return [$noRows, NULL];
4628       }
4629       $val = $query->store($dao);
4630       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4631
4632       if (!empty($convertedVals)) {
4633         $val = array_replace_recursive($val, $convertedVals);
4634       }
4635       $values[$dao->$entityIDField] = $val;
4636     }
4637     return [$values, $options];
4638   }
4639
4640   /**
4641    * Get the actual custom field name by stripping off the appended string.
4642    *
4643    * The string could be _relative, _from, or _to
4644    *
4645    * @todo use metadata rather than convention to do this.
4646    *
4647    * @param string $parameterName
4648    *   The name of the parameter submitted to the form.
4649    *   e.g
4650    *   custom_3_relative
4651    *   custom_3_from
4652    *
4653    * @return string
4654    */
4655   public static function getCustomFieldName($parameterName) {
4656     if (substr($parameterName, -5, 5) == '_from') {
4657       return substr($parameterName, 0, strpos($parameterName, '_from'));
4658     }
4659     if (substr($parameterName, -9, 9) == '_relative') {
4660       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4661     }
4662     if (substr($parameterName, -3, 3) == '_to') {
4663       return substr($parameterName, 0, strpos($parameterName, '_to'));
4664     }
4665   }
4666
4667   /**
4668    * Convert submitted values for relative custom fields to query object format.
4669    *
4670    * The query will support the sqlOperator format so convert to that format.
4671    *
4672    * @param array $formValues
4673    *   Submitted values.
4674    * @param array $params
4675    *   Converted parameters for the query object.
4676    * @param string $values
4677    *   Submitted value.
4678    * @param string $fieldName
4679    *   Submitted field name. (Matches form field not DB field.)
4680    */
4681   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4682     if (empty($values)) {
4683       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4684       // this function gets called again after they fields have been converted which can get ugly.
4685       return;
4686     }
4687     $customFieldName = self::getCustomFieldName($fieldName);
4688
4689     if (substr($fieldName, -9, 9) == '_relative') {
4690       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4691     }
4692     else {
4693       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4694         // Both to & from are set. We only need to acton one, choosing from.
4695         return;
4696       }
4697
4698       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4699       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4700
4701       if (self::isCustomDateField($customFieldName)) {
4702         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4703       }
4704     }
4705
4706     if ($from) {
4707       if ($to) {
4708         $relativeFunction = ['BETWEEN' => [$from, $to]];
4709       }
4710       else {
4711         $relativeFunction = ['>=' => $from];
4712       }
4713     }
4714     else {
4715       $relativeFunction = ['<=' => $to];
4716     }
4717     $params[] = [
4718       $customFieldName,
4719       '=',
4720       $relativeFunction,
4721       0,
4722       0,
4723     ];
4724   }
4725
4726   /**
4727    * Are we dealing with custom field of type date.
4728    *
4729    * @param $fieldName
4730    *
4731    * @return bool
4732    * @throws \CiviCRM_API3_Exception
4733    */
4734   public static function isCustomDateField($fieldName) {
4735     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4736       return FALSE;
4737     }
4738     try {
4739       $customFieldDataType = civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type']);
4740       if ('Date' == $customFieldDataType) {
4741         return TRUE;
4742       }
4743     }
4744     catch (CiviCRM_API3_Exception $e) {
4745     }
4746     return FALSE;
4747   }
4748
4749   /**
4750    * Has this field already been reformatting to Query object syntax.
4751    *
4752    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4753    * to come with associated double queries & is possibly damaging performance.
4754    *
4755    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4756    * them as they are.
4757    *
4758    * @param mixed $values
4759    *   Value in formValues for the field.
4760    *
4761    * @return bool;
4762    */
4763   public static function isAlreadyProcessedForQueryFormat($values) {
4764     if (!is_array($values)) {
4765       return FALSE;
4766     }
4767     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4768       return FALSE;
4769     }
4770     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4771   }
4772
4773   /**
4774    * If the state and country are passed remove state.
4775    *
4776    * Country is implicit from the state, but including both results in
4777    * a poor query as there is no combined index on state AND country.
4778    *
4779    * CRM-18125
4780    *
4781    * @param array $formValues
4782    */
4783   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4784     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4785       // The use of array map sanitises the data by ensuring we are dealing with integers.
4786       $states = implode(', ', array_map('intval', $formValues['state_province']));
4787       $countryList = CRM_Core_DAO::singleValueQuery(
4788         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4789       );
4790       if ($countryList == $formValues['country']) {
4791         unset($formValues['country']);
4792       }
4793     }
4794   }
4795
4796   /**
4797    * For some special cases, grouping by subset of select fields becomes mandatory.
4798    * Hence, full_group_by mode is handled by appending any_value
4799    * keyword to select fields not present in groupBy
4800    *
4801    * @param array $selectClauses
4802    * @param array $groupBy - Columns already included in GROUP By clause.
4803    * @param string $aggregateFunction
4804    *
4805    * @return string
4806    */
4807   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4808     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4809       $groupBy = array_map('trim', (array) $groupBy);
4810       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4811       foreach ($selectClauses as $key => &$val) {
4812         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4813         // append ANY_VALUE() keyword
4814         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4815           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4816             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4817             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4818         }
4819       }
4820     }
4821
4822     return "SELECT " . implode(', ', $selectClauses) . " ";
4823   }
4824
4825   /**
4826    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4827    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4828    * keyword to select fields not present in groupBy
4829    *
4830    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4831    * @param array $orderBys - ORDER BY sub-clauses
4832    *
4833    */
4834   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4835     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4836       foreach ($orderBys as $orderBy) {
4837         // remove sort syntax from ORDER BY clauses if present
4838         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4839         // if ORDER BY column is not present in GROUP BY then append it to end
4840         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4841           $groupBy .= ", {$orderBy}";
4842         }
4843       }
4844     }
4845   }
4846
4847   /**
4848    * Include Select columns in groupBy clause.
4849    *
4850    * @param array $selectClauses
4851    * @param array $groupBy - Columns already included in GROUP By clause.
4852    *
4853    * @return string
4854    */
4855   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4856     $groupBy = (array) $groupBy;
4857     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4858     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4859
4860     //return if ONLY_FULL_GROUP_BY is not enabled.
4861     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4862       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4863       foreach ($selectClauses as $key => $val) {
4864         $aliasArray = preg_split('/ as /i', $val);
4865         // if more than 1 alias we need to split by ','.
4866         if (count($aliasArray) > 2) {
4867           $aliasArray = preg_split('/,/', $val);
4868           foreach ($aliasArray as $key => $value) {
4869             $alias = current(preg_split('/ as /i', $value));
4870             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4871               $groupBy[] = $alias;
4872             }
4873           }
4874         }
4875         else {
4876           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4877           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4878           $tableName = current(explode('.', $selectColumn));
4879           $primaryKey = "{$tableName}.id";
4880           // exclude columns which are already included in groupBy and aggregate functions from select
4881           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4882           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4883             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4884               $groupBy[] = $alias;
4885             }
4886             else {
4887               $groupBy[] = $selectColumn;
4888             }
4889           }
4890         }
4891       }
4892     }
4893
4894     if (!empty($groupBy)) {
4895       return " GROUP BY " . implode(', ', $groupBy);
4896     }
4897     return '';
4898   }
4899
4900   /**
4901    * Create and query the db for an contact search.
4902    *
4903    * @param int $offset
4904    *   The offset for the query.
4905    * @param int $rowCount
4906    *   The number of rows to return.
4907    * @param string|CRM_Utils_Sort $sort
4908    *   The order by string.
4909    * @param bool $count
4910    *   Is this a count only query ?.
4911    * @param bool $includeContactIds
4912    *   Should we include contact ids?.
4913    * @param bool $sortByChar
4914    *   If true returns the distinct array of first characters for search results.
4915    * @param bool $groupContacts
4916    *   If true, return only the contact ids.
4917    * @param bool $returnQuery
4918    *   Should we return the query as a string.
4919    * @param string $additionalWhereClause
4920    *   If the caller wants to further restrict the search (used for components).
4921    * @param null $sortOrder
4922    * @param string $additionalFromClause
4923    *   Should be clause with proper joins, effective to reduce where clause load.
4924    *
4925    * @param bool $skipOrderAndLimit
4926    *
4927    * @return CRM_Core_DAO
4928    */
4929   public function searchQuery(
4930     $offset = 0, $rowCount = 0, $sort = NULL,
4931     $count = FALSE, $includeContactIds = FALSE,
4932     $sortByChar = FALSE, $groupContacts = FALSE,
4933     $returnQuery = FALSE,
4934     $additionalWhereClause = NULL, $sortOrder = NULL,
4935     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4936   ) {
4937
4938     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4939
4940     if ($returnQuery) {
4941       return $query;
4942     }
4943     if ($count) {
4944       return CRM_Core_DAO::singleValueQuery($query);
4945     }
4946
4947     $dao = CRM_Core_DAO::executeQuery($query);
4948
4949     // We can always call this - it will only re-enable if it was originally enabled.
4950     CRM_Core_DAO::reenableFullGroupByMode();
4951
4952     if ($groupContacts) {
4953       $ids = [];
4954       while ($dao->fetch()) {
4955         $ids[] = $dao->id;
4956       }
4957       return implode(',', $ids);
4958     }
4959
4960     return $dao;
4961   }
4962
4963   /**
4964    * Create and query the db for the list of all first letters used by contacts
4965    *
4966    * @return CRM_Core_DAO
4967    */
4968   public function alphabetQuery() {
4969     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
4970     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
4971       {$sqlParts['from']}
4972       {$sqlParts['where']}
4973       {$sqlParts['having']}
4974       GROUP BY sort_name
4975       ORDER BY sort_name asc";
4976     $dao = CRM_Core_DAO::executeQuery($query);
4977     return $dao;
4978   }
4979
4980   /**
4981    * Fetch a list of contacts for displaying a search results page
4982    *
4983    * @param array $cids
4984    *   List of contact IDs
4985    * @param bool $includeContactIds
4986    * @return CRM_Core_DAO
4987    */
4988   public function getCachedContacts($cids, $includeContactIds) {
4989     CRM_Core_DAO::disableFullGroupByMode();
4990     CRM_Utils_Type::validateAll($cids, 'Positive');
4991     $this->_includeContactIds = $includeContactIds;
4992     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
4993     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
4994     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
4995     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
4996     $order = 'ORDER BY _wgt';
4997     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
4998     $limit = '';
4999     $query = "$select $from $where $groupBy $order $limit";
5000
5001     $result = CRM_Core_DAO::executeQuery($query);
5002     CRM_Core_DAO::reenableFullGroupByMode();
5003     return $result;
5004   }
5005
5006   /**
5007    * Construct a SQL CASE expression.
5008    *
5009    * @param string $idCol
5010    *   The name of a column with ID's (eg 'contact_a.id').
5011    * @param array $cids
5012    *   Array(int $weight => int $id).
5013    * @return string
5014    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5015    */
5016   private function createSqlCase($idCol, $cids) {
5017     $buf = "CASE\n";
5018     foreach ($cids as $weight => $cid) {
5019       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5020     }
5021     $buf .= "END\n";
5022     return $buf;
5023   }
5024
5025   /**
5026    * Populate $this->_permissionWhereClause with permission related clause and update other
5027    * query related properties.
5028    *
5029    * Function calls ACL permission class and hooks to filter the query appropriately
5030    *
5031    * Note that these 2 params were in the code when extracted from another function
5032    * and a second round extraction would be to make them properties of the class
5033    *
5034    * @param bool $onlyDeleted
5035    *   Only get deleted contacts.
5036    * @param bool $count
5037    *   Return Count only.
5038    */
5039   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5040     if (!$this->_skipPermission) {
5041       $this->_permissionWhereClause = CRM_ACL_API::whereClause(
5042         CRM_Core_Permission::VIEW,
5043         $this->_tables,
5044         $this->_whereTables,
5045         NULL,
5046         $onlyDeleted,
5047         $this->_skipDeleteClause
5048       );
5049
5050       // regenerate fromClause since permission might have added tables
5051       if ($this->_permissionWhereClause) {
5052         //fix for row count in qill (in contribute/membership find)
5053         if (!$count) {
5054           $this->_useDistinct = TRUE;
5055         }
5056         //CRM-15231
5057         $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
5058         $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
5059         // note : this modifies _fromClause and _simpleFromClause
5060         $this->includePseudoFieldsJoin($this->_sort);
5061       }
5062     }
5063     else {
5064       // add delete clause if needed even if we are skipping permission
5065       // CRM-7639
5066       if (!$this->_skipDeleteClause) {
5067         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5068           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5069         }
5070         else {
5071           // CRM-6181
5072           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5073         }
5074       }
5075     }
5076   }
5077
5078   /**
5079    * @param $val
5080    */
5081   public function setSkipPermission($val) {
5082     $this->_skipPermission = $val;
5083   }
5084
5085   /**
5086    * @param null $context
5087    *
5088    * @return array
5089    * @throws \CRM_Core_Exception
5090    */
5091   public function summaryContribution($context = NULL) {
5092     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5093     if ($this->_permissionWhereClause) {
5094       $where .= " AND " . $this->_permissionWhereClause;
5095     }
5096     if ($context == 'search') {
5097       $where .= " AND contact_a.is_deleted = 0 ";
5098     }
5099
5100     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5101
5102     $summary = ['total' => []];
5103     $this->addBasicStatsToSummary($summary, $where, $from);
5104
5105     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5106       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5107     }
5108
5109     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5110
5111     return $summary;
5112   }
5113
5114   /**
5115    * Append financial ACL limits to the query from & where clauses, if applicable.
5116    *
5117    * @param string $where
5118    * @param string $from
5119    */
5120   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5121     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5122       return;
5123     }
5124     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5125     if (!empty($financialTypes)) {
5126       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5127       $from .= " LEFT JOIN civicrm_line_item li
5128                   ON civicrm_contribution.id = li.contribution_id AND
5129                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5130     }
5131     else {
5132       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5133     }
5134   }
5135
5136   /**
5137    * Getter for the qill object.
5138    *
5139    * @return array
5140    */
5141   public function qill() {
5142     return $this->_qill;
5143   }
5144
5145   /**
5146    * Default set of return default hier return properties.
5147    *
5148    * @return array
5149    */
5150   public static function &defaultHierReturnProperties() {
5151     if (!isset(self::$_defaultHierReturnProperties)) {
5152       self::$_defaultHierReturnProperties = [
5153         'home_URL' => 1,
5154         'image_URL' => 1,
5155         'legal_identifier' => 1,
5156         'external_identifier' => 1,
5157         'contact_type' => 1,
5158         'contact_sub_type' => 1,
5159         'sort_name' => 1,
5160         'display_name' => 1,
5161         'nick_name' => 1,
5162         'first_name' => 1,
5163         'middle_name' => 1,
5164         'last_name' => 1,
5165         'prefix_id' => 1,
5166         'suffix_id' => 1,
5167         'formal_title' => 1,
5168         'communication_style_id' => 1,
5169         'email_greeting' => 1,
5170         'postal_greeting' => 1,
5171         'addressee' => 1,
5172         'birth_date' => 1,
5173         'gender_id' => 1,
5174         'preferred_communication_method' => 1,
5175         'do_not_phone' => 1,
5176         'do_not_email' => 1,
5177         'do_not_mail' => 1,
5178         'do_not_sms' => 1,
5179         'do_not_trade' => 1,
5180         'location' => [
5181           '1' => [
5182             'location_type' => 1,
5183             'street_address' => 1,
5184             'city' => 1,
5185             'state_province' => 1,
5186             'postal_code' => 1,
5187             'postal_code_suffix' => 1,
5188             'country' => 1,
5189             'phone-Phone' => 1,
5190             'phone-Mobile' => 1,
5191             'phone-Fax' => 1,
5192             'phone-1' => 1,
5193             'phone-2' => 1,
5194             'phone-3' => 1,
5195             'im-1' => 1,
5196             'im-2' => 1,
5197             'im-3' => 1,
5198             'email-1' => 1,
5199             'email-2' => 1,
5200             'email-3' => 1,
5201           ],
5202           '2' => [
5203             'location_type' => 1,
5204             'street_address' => 1,
5205             'city' => 1,
5206             'state_province' => 1,
5207             'postal_code' => 1,
5208             'postal_code_suffix' => 1,
5209             'country' => 1,
5210             'phone-Phone' => 1,
5211             'phone-Mobile' => 1,
5212             'phone-1' => 1,
5213             'phone-2' => 1,
5214             'phone-3' => 1,
5215             'im-1' => 1,
5216             'im-2' => 1,
5217             'im-3' => 1,
5218             'email-1' => 1,
5219             'email-2' => 1,
5220             'email-3' => 1,
5221           ],
5222         ],
5223       ];
5224     }
5225     return self::$_defaultHierReturnProperties;
5226   }
5227
5228   /**
5229    * Build query for a date field.
5230    *
5231    * @param array $values
5232    * @param string $tableName
5233    * @param string $fieldName
5234    * @param string $dbFieldName
5235    * @param string $fieldTitle
5236    * @param bool $appendTimeStamp
5237    * @param string $dateFormat
5238    * @param string|null $highDBFieldName
5239    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5240    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5241    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5242    *   date & less than the end date.
5243    *
5244    * @throws \CRM_Core_Exception
5245    */
5246   public function dateQueryBuilder(
5247     $values, $tableName, $fieldName,
5248     $dbFieldName, $fieldTitle,
5249     $appendTimeStamp = TRUE,
5250     $dateFormat = 'YmdHis',
5251     $highDBFieldName = NULL
5252   ) {
5253     // @todo - remove dateFormat - pretty sure it's never passed in...
5254     list($name, $op, $value, $grouping, $wildcard) = $values;
5255     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5256       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5257       return;
5258     }
5259     if ($tableName === 'civicrm_contact') {
5260       // Special handling for contact table as it has a known alias in advanced search.
5261       $tableName = 'contact_a';
5262     }
5263     if ($name === "{$fieldName}_low" ||
5264       $name === "{$fieldName}_high"
5265     ) {
5266       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5267         return;
5268       }
5269       $this->_rangeCache[$fieldName] = 1;
5270
5271       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5272
5273       if ($name == $fieldName . '_low') {
5274         $firstOP = '>=';
5275         $firstPhrase = ts('greater than or equal to');
5276         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5277
5278         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5279         if (!empty($secondValues) && $secondValues[2]) {
5280           $secondOP = '<=';
5281           $secondPhrase = ts('less than or equal to');
5282           $secondValue = $secondValues[2];
5283
5284           if ($appendTimeStamp && strlen($secondValue) == 10) {
5285             $secondValue .= ' 23:59:59';
5286           }
5287           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5288         }
5289       }
5290       elseif ($name == $fieldName . '_high') {
5291         $firstOP = '<=';
5292         $firstPhrase = ts('less than or equal to');
5293
5294         if ($appendTimeStamp && strlen($value) == 10) {
5295           $value .= ' 23:59:59';
5296         }
5297         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5298
5299         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5300         if (!empty($secondValues) && $secondValues[2]) {
5301           $secondOP = '>=';
5302           $secondPhrase = ts('greater than or equal to');
5303           $secondValue = $secondValues[2];
5304           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5305         }
5306       }
5307
5308       if (!$appendTimeStamp) {
5309         $firstDate = substr($firstDate, 0, 8);
5310       }
5311       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5312
5313       if ($secondDate) {
5314         if (!$appendTimeStamp) {
5315           $secondDate = substr($secondDate, 0, 8);
5316         }
5317         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5318       }
5319
5320       if ($secondDate) {
5321         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5322         $this->_where[$grouping][] = "
5323 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5324 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5325 ";
5326         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5327       }
5328       else {
5329         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5330         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5331       }
5332     }
5333
5334     if ($name == $fieldName) {
5335       //In Get API, for operators other then '=' the $value is in array(op => value) format
5336       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5337         $op = key($value);
5338         $value = $value[$op];
5339       }
5340
5341       $date = $format = NULL;
5342       if (strstr($op, 'IN')) {
5343         $format = [];
5344         foreach ($value as &$date) {
5345           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5346           if (!$appendTimeStamp) {
5347             $date = substr($date, 0, 8);
5348           }
5349           $format[] = CRM_Utils_Date::customFormat($date);
5350         }
5351         $date = "('" . implode("','", $value) . "')";
5352         $format = implode(', ', $format);
5353       }
5354       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5355         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5356         if (!$appendTimeStamp) {
5357           $date = substr($date, 0, 8);
5358         }
5359         $format = CRM_Utils_Date::customFormat($date);
5360         $date = "'$date'";
5361       }
5362
5363       if ($date) {
5364         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5365       }
5366       else {
5367         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5368       }
5369
5370       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5371       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5372     }
5373
5374     // Ensure the tables are set, but don't whomp anything.
5375     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5376     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5377   }
5378
5379   /**
5380    * @param $values
5381    * @param string $tableName
5382    * @param string $fieldName
5383    * @param string $dbFieldName
5384    * @param $fieldTitle
5385    * @param null $options
5386    */
5387   public function numberRangeBuilder(
5388     &$values,
5389     $tableName, $fieldName,
5390     $dbFieldName, $fieldTitle,
5391     $options = NULL
5392   ) {
5393     list($name, $op, $value, $grouping, $wildcard) = $values;
5394
5395     if ($name == "{$fieldName}_low" ||
5396       $name == "{$fieldName}_high"
5397     ) {
5398       if (isset($this->_rangeCache[$fieldName])) {
5399         return;
5400       }
5401       $this->_rangeCache[$fieldName] = 1;
5402
5403       $secondOP = $secondPhrase = $secondValue = NULL;
5404
5405       if ($name == "{$fieldName}_low") {
5406         $firstOP = '>=';
5407         $firstPhrase = ts('greater than');
5408
5409         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5410         if (!empty($secondValues)) {
5411           $secondOP = '<=';
5412           $secondPhrase = ts('less than');
5413           $secondValue = $secondValues[2];
5414         }
5415       }
5416       else {
5417         $firstOP = '<=';
5418         $firstPhrase = ts('less than');
5419
5420         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5421         if (!empty($secondValues)) {
5422           $secondOP = '>=';
5423           $secondPhrase = ts('greater than');
5424           $secondValue = $secondValues[2];
5425         }
5426       }
5427
5428       if ($secondOP) {
5429         $this->_where[$grouping][] = "
5430 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5431 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5432 ";
5433         $displayValue = $options ? $options[$value] : $value;
5434         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5435
5436         $this->_qill[$grouping][]
5437           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5438       }
5439       else {
5440         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5441         $displayValue = $options ? $options[$value] : $value;
5442         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5443       }
5444       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5445
5446       return;
5447     }
5448
5449     if ($name == $fieldName) {
5450       $op = '=';
5451       $phrase = '=';
5452
5453       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5454
5455       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5456       $displayValue = $options ? $options[$value] : $value;
5457       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5458     }
5459   }
5460
5461   /**
5462    * @param $values
5463    * @param string $tableName
5464    * @param string $fieldName
5465    * @param string $dbFieldName
5466    * @param $fieldTitle
5467    * @param null $options
5468    */
5469   public function ageRangeQueryBuilder(
5470     &$values,
5471     $tableName, $fieldName,
5472     $dbFieldName, $fieldTitle,
5473     $options = NULL
5474   ) {
5475     list($name, $op, $value, $grouping, $wildcard) = $values;
5476
5477     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5478     // will be treated as current day
5479     $asofDate = NULL;
5480     if ($asofDateValues) {
5481       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5482       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5483       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5484     }
5485
5486     if ($name == "{$fieldName}_low" ||
5487       $name == "{$fieldName}_high"
5488     ) {
5489       if (isset($this->_rangeCache[$fieldName])) {
5490         return;
5491       }
5492       $this->_rangeCache[$fieldName] = 1;
5493
5494       $secondOP = $secondPhrase = $secondValue = NULL;
5495
5496       if ($name == "{$fieldName}_low") {
5497         $firstPhrase = ts('greater than or equal to');
5498         // NB: age > X means date of birth < Y
5499         $firstOP = '<=';
5500         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5501
5502         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5503         if (!empty($secondValues)) {
5504           $secondOP = '>=';
5505           $secondPhrase = ts('less than or equal to');
5506           $secondValue = $secondValues[2];
5507           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5508         }
5509       }
5510       else {
5511         $firstOP = '>=';
5512         $firstPhrase = ts('less than or equal to');
5513         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5514
5515         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5516         if (!empty($secondValues)) {
5517           $secondOP = '<=';
5518           $secondPhrase = ts('greater than or equal to');
5519           $secondValue = $secondValues[2];
5520           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5521         }
5522       }
5523
5524       if ($secondOP) {
5525         $this->_where[$grouping][] = "
5526 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5527 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5528 ";
5529         $displayValue = $options ? $options[$value] : $value;
5530         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5531
5532         $this->_qill[$grouping][]
5533           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5534       }
5535       else {
5536         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5537         $displayValue = $options ? $options[$value] : $value;
5538         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5539       }
5540       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5541       return;
5542     }
5543   }
5544
5545   /**
5546    * Calculate date from age.
5547    *
5548    * @param string $asofDate
5549    * @param int $age
5550    * @param string $type
5551    *
5552    * @return string
5553    * @throws \Exception
5554    */
5555   public static function calcDateFromAge($asofDate, $age, $type) {
5556     $date = new DateTime($asofDate);
5557     if ($type == "min") {
5558       // minimum age is $age: dob <= date - age "235959"
5559       $date->sub(new DateInterval("P" . $age . "Y"));
5560       return $date->format('Ymd') . "235959";
5561     }
5562     else {
5563       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5564       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5565       return $date->format('Ymd') . "000000";
5566     }
5567   }
5568
5569   /**
5570    * Given the field name, operator, value & its data type
5571    * builds the where Clause for the query
5572    * used for handling 'IS NULL'/'IS NOT NULL' operators
5573    *
5574    * @param string $field
5575    *   Fieldname.
5576    * @param string $op
5577    *   Operator.
5578    * @param string $value
5579    *   Value.
5580    * @param string $dataType
5581    *   Data type of the field.
5582    *
5583    * @return string
5584    *   Where clause for the query.
5585    * @throws \CRM_Core_Exception
5586    */
5587   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5588     $op = trim($op);
5589     $clause = "$field $op";
5590
5591     switch ($op) {
5592       case 'IS NULL':
5593       case 'IS NOT NULL':
5594         return $clause;
5595
5596       case 'IS EMPTY':
5597         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5598         return $clause;
5599
5600       case 'IS NOT EMPTY':
5601         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5602         return $clause;
5603
5604       case 'RLIKE':
5605         return " {$clause} BINARY '{$value}' ";
5606
5607       case 'IN':
5608       case 'NOT IN':
5609         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5610         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5611           $value = [$op => (array) $value];
5612         }
5613
5614       default:
5615         if (empty($dataType) || $dataType == 'Date') {
5616           $dataType = 'String';
5617         }
5618         if (is_array($value)) {
5619           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5620           // widely used and consistent across the codebase
5621           // adding this here won't accept the search functions which don't submit an array
5622           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5623
5624             return $queryString;
5625           }
5626           if (!empty($value[0]) && $op === 'BETWEEN') {
5627             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5628             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5629               return $queryString;
5630             }
5631           }
5632           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5633         }
5634
5635         $value = CRM_Utils_Type::escape($value, $dataType);
5636         // if we don't have a dataType we should assume
5637         if ($dataType == 'String' || $dataType == 'Text') {
5638           $value = "'" . $value . "'";
5639         }
5640         return "$clause $value";
5641     }
5642   }
5643
5644   /**
5645    * @param bool $reset
5646    *
5647    * @return array
5648    */
5649   public function openedSearchPanes($reset = FALSE) {
5650     if (!$reset || empty($this->_whereTables)) {
5651       return self::$_openedPanes;
5652     }
5653
5654     // pane name to table mapper
5655     $panesMapper = [
5656       ts('Contributions') => 'civicrm_contribution',
5657       ts('Memberships') => 'civicrm_membership',
5658       ts('Events') => 'civicrm_participant',
5659       ts('Relationships') => 'civicrm_relationship',
5660       ts('Activities') => 'civicrm_activity',
5661       ts('Pledges') => 'civicrm_pledge',
5662       ts('Cases') => 'civicrm_case',
5663       ts('Grants') => 'civicrm_grant',
5664       ts('Address Fields') => 'civicrm_address',
5665       ts('Notes') => 'civicrm_note',
5666       ts('Change Log') => 'civicrm_log',
5667       ts('Mailings') => 'civicrm_mailing',
5668     ];
5669     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5670
5671     foreach (array_keys($this->_whereTables) as $table) {
5672       if ($panName = array_search($table, $panesMapper)) {
5673         self::$_openedPanes[$panName] = TRUE;
5674       }
5675     }
5676
5677     return self::$_openedPanes;
5678   }
5679
5680   /**
5681    * @param $operator
5682    */
5683   public function setOperator($operator) {
5684     $validOperators = ['AND', 'OR'];
5685     if (!in_array($operator, $validOperators)) {
5686       $operator = 'AND';
5687     }
5688     $this->_operator = $operator;
5689   }
5690
5691   /**
5692    * @return string
5693    */
5694   public function getOperator() {
5695     return $this->_operator;
5696   }
5697
5698   /**
5699    * @param $from
5700    * @param $where
5701    * @param $having
5702    */
5703   public function filterRelatedContacts(&$from, &$where, &$having) {
5704     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5705       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5706     }
5707     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5708     // since there only can be one instance of this filter in every query
5709     // skip if filter has already applied
5710     foreach ($_rTempCache as $acache) {
5711       foreach ($acache['queries'] as $aqcache) {
5712         if (strpos($from, $aqcache['from']) !== FALSE) {
5713           $having = NULL;
5714           return;
5715         }
5716       }
5717     }
5718     $arg_sig = sha1("$from $where $having");
5719     if (isset($_rTempCache[$arg_sig])) {
5720       $cache = $_rTempCache[$arg_sig];
5721     }
5722     else {
5723       // create temp table with contact ids
5724
5725       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5726
5727       $sql = "
5728 REPLACE INTO $tableName ( contact_id )
5729 SELECT contact_a.id
5730        $from
5731        $where
5732        $having
5733 ";
5734       CRM_Core_DAO::executeQuery($sql);
5735
5736       $cache = ['tableName' => $tableName, 'queries' => []];
5737       $_rTempCache[$arg_sig] = $cache;
5738     }
5739     // upsert the query depending on relationship type
5740     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5741       $qcache = $cache['queries'][$this->_displayRelationshipType];
5742     }
5743     else {
5744       $tableName = $cache['tableName'];
5745       $qcache = [
5746         "from" => "",
5747         "where" => "",
5748       ];
5749       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5750       if (is_numeric($this->_displayRelationshipType)) {
5751         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5752         $qcache['from'] = "
5753 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5754 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5755 ";
5756         $qcache['where'] = "
5757 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5758 AND   displayRelType.is_active = 1
5759 ";
5760       }
5761       else {
5762         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5763         if ($dirOne == 'a') {
5764           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5765           $qcache['from'] .= "
5766 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5767 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5768 ";
5769         }
5770         else {
5771           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5772           $qcache['from'] .= "
5773 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5774 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5775 ";
5776         }
5777         $qcache['where'] = "
5778 WHERE displayRelType.relationship_type_id = $relType
5779 AND   displayRelType.is_active = 1
5780 ";
5781       }
5782       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5783       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5784     }
5785     $qillMessage = ts('Contacts with a Relationship Type of: ');
5786     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5787     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5788       $this->_qill[0][] = $iqill;
5789     }
5790     if (strpos($from, $qcache['from']) === FALSE) {
5791       // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5792       // this happens when we have an event_type in the quert (CRM-7969)
5793       $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5794       $from .= $qcache['from'];
5795       $where = $qcache['where'];
5796       if (!empty($this->_permissionWhereClause)) {
5797         $where .= "AND $this->_permissionWhereClause";
5798       }
5799     }
5800
5801     $having = NULL;
5802   }
5803
5804   /**
5805    * See CRM-19811 for why this is database hurty without apparent benefit.
5806    *
5807    * @param $op
5808    *
5809    * @return bool
5810    */
5811   public static function caseImportant($op) {
5812     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5813   }
5814
5815   /**
5816    * @param $returnProperties
5817    * @param $prefix
5818    *
5819    * @return bool
5820    */
5821   public static function componentPresent(&$returnProperties, $prefix) {
5822     foreach ($returnProperties as $name => $dontCare) {
5823       if (substr($name, 0, strlen($prefix)) == $prefix) {
5824         return TRUE;
5825       }
5826     }
5827     return FALSE;
5828   }
5829
5830   /**
5831    * Builds the necessary structures for all fields that are similar to option value look-ups.
5832    *
5833    * @param string $name
5834    *   the name of the field.
5835    * @param string $op
5836    *   the sql operator, this function should handle ALL SQL operators.
5837    * @param string $value
5838    *   depends on the operator and who's calling the query builder.
5839    * @param int $grouping
5840    *   the index where to place the where clause.
5841    * @param string $daoName
5842    *   DAO Name.
5843    * @param array $field
5844    *   an array that contains various properties of the field identified by $name.
5845    * @param string $label
5846    *   The label for this field element.
5847    * @param string $dataType
5848    *
5849    * @throws \CRM_Core_Exception
5850    */
5851   public function optionValueQuery(
5852     $name,
5853     $op,
5854     $value,
5855     $grouping,
5856     $daoName = NULL,
5857     $field,
5858     $label,
5859     $dataType = 'String'
5860   ) {
5861
5862     $pseudoFields = [
5863       'email_greeting',
5864       'postal_greeting',
5865       'addressee',
5866     ];
5867
5868     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5869     if ($tableName == 'civicrm_contact') {
5870       $wc = "contact_a.$fieldName";
5871     }
5872     else {
5873       // Special handling for on_hold, so that we actually use the 'where'
5874       // property in order to limit the query by the on_hold status of the email,
5875       // instead of using email.id which would be nonsensical.
5876       if ($field['name'] === 'on_hold') {
5877         $wc = $field['where'];
5878       }
5879       else {
5880         $wc = "$tableName.id";
5881       }
5882     }
5883
5884     if (in_array($name, $pseudoFields)) {
5885       $wc = "contact_a.{$name}_id";
5886       $dataType = 'Positive';
5887       $value = (!$value) ? 0 : $value;
5888     }
5889     if ($name == "world_region") {
5890       $field['name'] = $name;
5891     }
5892
5893     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5894     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5895     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5896   }
5897
5898   /**
5899    * Check and explode a user defined numeric string into an array
5900    * this was the protocol used by search builder in the old old days before we had
5901    * super nice js widgets to do the hard work
5902    *
5903    * @param string $string
5904    * @param string $dataType
5905    *   The dataType we should check for the values, default integer.
5906    *
5907    * @return bool|array
5908    *   false if string does not match the pattern
5909    *   array of numeric values if string does match the pattern
5910    */
5911   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5912     $string = trim($string);
5913     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5914       return FALSE;
5915     }
5916
5917     $string = substr($string, 1, -1);
5918     $values = explode(',', $string);
5919     if (empty($values)) {
5920       return FALSE;
5921     }
5922
5923     $returnValues = [];
5924     foreach ($values as $v) {
5925       if ($dataType == 'Integer' && !is_numeric($v)) {
5926         return FALSE;
5927       }
5928       elseif ($dataType == 'String' && !is_string($v)) {
5929         return FALSE;
5930       }
5931       $returnValues[] = trim($v);
5932     }
5933
5934     if (empty($returnValues)) {
5935       return FALSE;
5936     }
5937
5938     return $returnValues;
5939   }
5940
5941   /**
5942    * Convert the pseudo constants id's to their names
5943    *
5944    * @param CRM_Core_DAO $dao
5945    * @param bool $return
5946    * @param bool $usedForAPI
5947    *
5948    * @return array|NULL
5949    */
5950   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
5951     if (empty($this->_pseudoConstantsSelect)) {
5952       return NULL;
5953     }
5954     $values = [];
5955     foreach ($this->_pseudoConstantsSelect as $key => $value) {
5956       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
5957         continue;
5958       }
5959
5960       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
5961         $val = $dao->{$value['idCol']};
5962         if ($key == 'groups') {
5963           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
5964           continue;
5965         }
5966
5967         if (CRM_Utils_System::isNull($val)) {
5968           $dao->$key = NULL;
5969         }
5970         elseif (!empty($value['pseudoconstant'])) {
5971           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
5972           // and we can use the metadata to figure it out.
5973           // ideally this bit of IF will absorb & replace all the rest in time as we move to
5974           // more metadata based choices.
5975           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
5976             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
5977             foreach ($dbValues as $pseudoValue) {
5978               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
5979             }
5980
5981             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
5982             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
5983             if ($usedForAPI && $realFieldName) {
5984               // normally we would see 2 fields returned for pseudoConstants. An exception is
5985               // preferred_communication_method where there is no id-variant.
5986               // For the api we prioritise getting the real data returned.
5987               // over the resolved version
5988               $dao->$realFieldName = $dbValues;
5989             }
5990
5991           }
5992           else {
5993             // This is basically the same as the default but since we have the bao we can use
5994             // a cached function.
5995             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
5996           }
5997         }
5998         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
5999           //preserve id value
6000           $idColumn = "{$key}_id";
6001           $dao->$idColumn = $val;
6002
6003           if ($key == 'state_province_name') {
6004             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6005           }
6006           else {
6007             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6008           }
6009         }
6010         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6011           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6012         }
6013         // @todo handle this in the section above for pseudoconstants.
6014         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6015           // @todo define bao on this & merge into the above condition.
6016           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6017
6018           if ($value['pseudoField'] == 'participant_role') {
6019             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6020             foreach ($viewValues as $k => $v) {
6021               $viewValues[$k] = $pseudoOptions[$v];
6022             }
6023           }
6024           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6025         }
6026         else {
6027           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6028           $dao->$key = $labels[$val];
6029         }
6030
6031         // return converted values in array format
6032         if ($return) {
6033           if (strpos($key, '-') !== FALSE) {
6034             $keyVal = explode('-', $key);
6035             $current = &$values;
6036             $lastElement = array_pop($keyVal);
6037             foreach ($keyVal as $v) {
6038               if (!array_key_exists($v, $current)) {
6039                 $current[$v] = [];
6040               }
6041               $current = &$current[$v];
6042             }
6043             $current[$lastElement] = $dao->$key;
6044           }
6045           else {
6046             $values[$key] = $dao->$key;
6047           }
6048         }
6049       }
6050     }
6051     if (!$usedForAPI) {
6052       foreach ($this->legacyHackedFields as $realField => $labelField) {
6053         // This is a temporary routine for handling these fields while
6054         // we figure out how to handled them based on metadata in
6055         /// export and search builder. CRM-19815, CRM-19830.
6056         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6057           $dao->$realField = $dao->$labelField;
6058         }
6059       }
6060     }
6061     return $values;
6062   }
6063
6064   /**
6065    * Include pseudo fields LEFT JOIN.
6066    * @param string|array $sort can be a object or string
6067    *
6068    * @return array|NULL
6069    */
6070   public function includePseudoFieldsJoin($sort) {
6071     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6072       return NULL;
6073     }
6074     $sort = is_string($sort) ? $sort : $sort->orderBy();
6075     $present = [];
6076
6077     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6078       if (!empty($value['table'])) {
6079         $regex = "/({$value['table']}\.|{$name})/";
6080         if (preg_match($regex, $sort)) {
6081           $this->_elemnt[$value['element']] = 1;
6082           $this->_select[$value['element']] = $value['select'];
6083           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6084           $present[$value['table']] = $value['join'];
6085         }
6086       }
6087     }
6088     $presentSimpleFrom = $present;
6089
6090     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6091       array_key_exists('civicrm_country', $presentSimpleFrom)
6092     ) {
6093       unset($presentSimpleFrom['civicrm_country']);
6094     }
6095     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6096       array_key_exists('civicrm_country', $present)
6097     ) {
6098       unset($present['civicrm_country']);
6099     }
6100
6101     $presentClause = $presentSimpleFromClause = NULL;
6102     if (!empty($present)) {
6103       $presentClause = implode(' ', $present);
6104     }
6105     if (!empty($presentSimpleFrom)) {
6106       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6107     }
6108
6109     $this->_fromClause = $this->_fromClause . $presentClause;
6110     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6111
6112     return [$presentClause, $presentSimpleFromClause];
6113   }
6114
6115   /**
6116    * Build qill for field.
6117    *
6118    * Qill refers to the query detail visible on the UI.
6119    *
6120    * @param string $daoName
6121    * @param string $fieldName
6122    * @param mixed $fieldValue
6123    * @param string $op
6124    * @param array $pseudoExtraParam
6125    * @param int $type
6126    *   Type of the field per CRM_Utils_Type
6127    *
6128    * @return array
6129    */
6130   public static function buildQillForFieldValue(
6131     $daoName,
6132     $fieldName,
6133     $fieldValue,
6134     $op,
6135     $pseudoExtraParam = [],
6136     $type = CRM_Utils_Type::T_STRING
6137   ) {
6138     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6139
6140     //API usually have fieldValue format as array(operator => array(values)),
6141     //so we need to separate operator out of fieldValue param
6142     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6143       $op = key($fieldValue);
6144       $fieldValue = $fieldValue[$op];
6145     }
6146
6147     // if Operator chosen is NULL/EMPTY then
6148     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6149       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6150     }
6151
6152     // @todo - if the right BAO is passed in special handling for the below
6153     // fields should not be required. testQillOptions.
6154     if ($fieldName == 'country_id') {
6155       $pseudoOptions = CRM_Core_PseudoConstant::country();
6156     }
6157     elseif ($fieldName == 'county_id') {
6158       $pseudoOptions = CRM_Core_PseudoConstant::county();
6159     }
6160     elseif ($fieldName == 'world_region') {
6161       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6162     }
6163     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6164       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6165       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6166     }
6167     elseif ($fieldName == 'contribution_product_id') {
6168       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6169     }
6170     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6171       $pseudoOptions = CRM_Core_PseudoConstant::group();
6172     }
6173     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6174       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6175     }
6176     elseif ($daoName) {
6177       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6178     }
6179
6180     if (is_array($fieldValue)) {
6181       $qillString = [];
6182       if (!empty($pseudoOptions)) {
6183         foreach ((array) $fieldValue as $val) {
6184           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6185         }
6186         $fieldValue = implode(', ', $qillString);
6187       }
6188       else {
6189         if ($type == CRM_Utils_Type::T_DATE) {
6190           foreach ($fieldValue as $index => $value) {
6191             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6192           }
6193         }
6194         $separator = ', ';
6195         // @todo - this is a bit specific (one operator).
6196         // However it is covered by a unit test so can be altered later with
6197         // some confidence.
6198         if ($op === 'BETWEEN') {
6199           $separator = ' AND ';
6200         }
6201         $fieldValue = implode($separator, $fieldValue);
6202       }
6203     }
6204     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6205       $fieldValue = $pseudoOptions[$fieldValue];
6206     }
6207     elseif ($type === CRM_Utils_Type::T_DATE) {
6208       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6209     }
6210
6211     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6212   }
6213
6214   /**
6215    * Get the qill (search description for field) for the specified field.
6216    *
6217    * @param string $daoName
6218    * @param string $name
6219    * @param string $value
6220    * @param string|array $op
6221    * @param string $label
6222    *
6223    * @return string
6224    */
6225   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6226     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6227     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6228   }
6229
6230   /**
6231    * Alter value to reflect wildcard settings.
6232    *
6233    * The form will have tried to guess whether this is a good field to wildcard but there is
6234    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6235    * or only the end of the string
6236    *
6237    * @param bool $wildcard
6238    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6239    * @param string $op
6240    *   Generally '=' or 'LIKE'.
6241    * @param string $value
6242    *   The search string.
6243    *
6244    * @return string
6245    */
6246   public static function getWildCardedValue($wildcard, $op, $value) {
6247     if ($wildcard && $op === 'LIKE') {
6248       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6249         return "%$value%";
6250       }
6251       else {
6252         return "$value%";
6253       }
6254     }
6255     else {
6256       return "$value";
6257     }
6258   }
6259
6260   /**
6261    * Process special fields of Search Form in OK (Operator in Key) format
6262    *
6263    * @param array $formValues
6264    * @param array $specialFields
6265    *    Special params to be processed
6266    * @param array $changeNames
6267    *   Array of fields whose name should be changed
6268    */
6269   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6270     // Array of special fields whose value are considered only for NULL or EMPTY operators
6271     $nullableFields = ['contribution_batch_id'];
6272
6273     foreach ($specialFields as $element) {
6274       $value = $formValues[$element] ?? NULL;
6275       if ($value) {
6276         if (is_array($value)) {
6277           if (in_array($element, array_keys($changeNames))) {
6278             unset($formValues[$element]);
6279             $element = $changeNames[$element];
6280           }
6281           $formValues[$element] = ['IN' => $value];
6282         }
6283         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6284           $formValues[$element] = [$value => 1];
6285         }
6286         elseif (!in_array($element, $nullableFields)) {
6287           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6288           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6289           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6290         }
6291       }
6292     }
6293   }
6294
6295   /**
6296    * Parse and assimilate the various sort options.
6297    *
6298    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6299    * `email`), the related table may be joined automatically.
6300    *
6301    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6302    * it's been extracted to a standalone function.
6303    *
6304    * @param string|CRM_Utils_Sort $sort
6305    *   The order by string.
6306    * @param null $sortOrder
6307    *   Who knows? Hu knows. He who knows Hu knows who.
6308    *
6309    * @return string
6310    *   list(string $orderByClause, string $additionalFromClause).
6311    *
6312    * @throws \CRM_Core_Exception
6313    */
6314   protected function prepareOrderBy($sort, $sortOrder) {
6315     $orderByArray = [];
6316     $orderBy = '';
6317
6318     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6319       isset($this->_distinctComponentClause)
6320     ) {
6321       if ($sort) {
6322         if (is_string($sort)) {
6323           $orderBy = $sort;
6324         }
6325         else {
6326           $orderBy = trim($sort->orderBy());
6327         }
6328         // Deliberately remove the backticks again, as they mess up the evil
6329         // string munging below. This balanced by re-escaping before use.
6330         $orderBy = str_replace('`', '', $orderBy);
6331
6332         if (!empty($orderBy)) {
6333           // this is special case while searching for
6334           // change log CRM-1718
6335           if (preg_match('/sort_name/i', $orderBy)) {
6336             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6337           }
6338
6339           if ($sortOrder) {
6340             $orderBy .= " $sortOrder";
6341           }
6342
6343           // always add contact_a.id to the ORDER clause
6344           // so the order is deterministic
6345           if (strpos('contact_a.id', $orderBy) === FALSE) {
6346             $orderBy .= ", contact_a.id";
6347           }
6348         }
6349       }
6350       else {
6351         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6352       }
6353     }
6354     if (!$orderBy) {
6355       return NULL;
6356     }
6357     // Remove this here & add it at the end for simplicity.
6358     $order = trim($orderBy);
6359     $orderByArray = explode(',', $order);
6360
6361     foreach ($orderByArray as $orderByClause) {
6362       $orderByClauseParts = explode(' ', trim($orderByClause));
6363       $field = $orderByClauseParts[0];
6364       $direction = $orderByClauseParts[1] ?? 'asc';
6365       $fieldSpec = $this->getMetadataForRealField($field);
6366
6367       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6368       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6369       // add keys to $this->fields to represent the actual keys for locations.
6370       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6371         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6372         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6373       }
6374
6375       if ($this->_returnProperties === []) {
6376         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6377           $this->_tables[$fieldSpec['table_name']] = 1;
6378           $order = $fieldSpec['where'] . ' ' . $direction;
6379         }
6380
6381       }
6382       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6383       // add to cfIDs array if not present
6384       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6385         $this->_cfIDs[$cfID] = [];
6386         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6387         $this->_customQuery->query();
6388         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6389         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6390       }
6391
6392       // By replacing the join to the option value table with the mysql construct
6393       // ORDER BY field('contribution_status_id', 2,1,4)
6394       // we can remove a join. In the case of the option value join it is
6395       /// a join known to cause slow queries.
6396       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6397       // matching an id+name parrern in the
6398       // first instance for scope reasons. They require slightly different handling as the column (label)
6399       // is not declared for them.
6400       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6401       // escaping.
6402       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6403       if (!empty($pseudoConstantMetadata)
6404       ) {
6405         if (!empty($pseudoConstantMetadata['optionGroupName'])
6406           || $this->isPseudoFieldAnFK($fieldSpec)
6407         ) {
6408           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6409           if (empty($fieldSpec['bao'])) {
6410             continue;
6411           }
6412           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name'], NULL, [
6413             'orderColumn' => CRM_Utils_Array::value('labelColumn', $pseudoConstantMetadata, 'label'),
6414           ]);
6415           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6416           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6417           // this might give some early warning.
6418           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6419           // use where if it's set to fully qualify ambiguous column names
6420           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6421           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6422           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6423         }
6424         //CRM-12565 add "`" around $field if it is a pseudo constant
6425         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6426         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6427           $order = str_replace($field, "`{$field}`", $order);
6428         }
6429       }
6430     }
6431
6432     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6433     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6434
6435     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6436     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6437     if ($order) {
6438       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6439       return ' ORDER BY ' . $order;
6440     }
6441   }
6442
6443   /**
6444    * Convert a string of group IDs to a string of group labels.
6445    *
6446    * The original string may include duplicates and groups the user does not have
6447    * permission to see.
6448    *
6449    * @param CRM_Core_DAO $dao
6450    * @param string $val
6451    *
6452    * @return string
6453    */
6454   public function convertGroupIDStringToLabelString(&$dao, $val) {
6455     $groupIDs = explode(',', $val);
6456     // Note that groups that the user does not have permission to will be excluded (good).
6457     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6458     return implode(', ', $groups);
6459
6460   }
6461
6462   /**
6463    * Set the qill and where properties for a field.
6464    *
6465    * This function is intended as a short-term function to encourage refactoring
6466    * & re-use - but really we should just have less special-casing.
6467    *
6468    * @param string $name
6469    * @param string $op
6470    * @param string|array $value
6471    * @param string $grouping
6472    * @param array $field
6473    *
6474    * @throws \CRM_Core_Exception
6475    */
6476   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6477     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6478     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6479     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6480       1 => $field['title'],
6481       2 => $qillop,
6482       3 => $qillVal,
6483     ]);
6484   }
6485
6486   /**
6487    * Has the pseudoconstant of the field been requested.
6488    *
6489    * For example if the field is payment_instrument_id then it
6490    * has been requested if either payment_instrument_id or payment_instrument
6491    * have been requested. Payment_instrument is the option groun name field value.
6492    *
6493    * @param array $field
6494    * @param string $fieldName
6495    *   The unique name of the field - ie. the one it will be aliased to in the query.
6496    *
6497    * @return bool
6498    */
6499   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6500     $realField = $this->getMetadataForRealField($fieldName);
6501     if (!isset($realField['pseudoconstant'])) {
6502       return FALSE;
6503     }
6504     $pseudoConstant = $realField['pseudoconstant'];
6505     if (empty($pseudoConstant['optionGroupName']) &&
6506       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6507       // We are increasing our pseudoconstant handling - but still very cautiously,
6508       // hence the check for labelColumn === name
6509       return FALSE;
6510     }
6511
6512     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6513       return TRUE;
6514     }
6515     if (!empty($this->_returnProperties[$fieldName])) {
6516       return TRUE;
6517     }
6518     // Is this still required - the above goes off the unique name. Test with things like
6519     // communication_preferences & prefix_id.
6520     if (!empty($this->_returnProperties[$field['name']])) {
6521       return TRUE;
6522     }
6523     return FALSE;
6524   }
6525
6526   /**
6527    * Get Select Clause.
6528    *
6529    * @return string
6530    */
6531   public function getSelect() {
6532     $select = 'SELECT ';
6533     if (isset($this->_distinctComponentClause)) {
6534       $select .= "{$this->_distinctComponentClause}, ";
6535     }
6536     $select .= implode(', ', $this->_select);
6537     return $select;
6538   }
6539
6540   /**
6541    * Add basic statistics to the summary.
6542    *
6543    * @param array $summary
6544    * @param string $where
6545    * @param string $from
6546    *
6547    * @return array
6548    * @throws \CRM_Core_Exception
6549    */
6550   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6551     $summary['total']['count'] = 0;
6552     $summary['total']['amount'] = $summary['total']['avg'] = [];
6553
6554     $query = "
6555       SELECT COUNT( conts.total_amount ) as total_count,
6556         SUM(   conts.total_amount ) as total_amount,
6557         AVG(   conts.total_amount ) as total_avg,
6558         conts.currency              as currency
6559       FROM (
6560         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6561         civicrm_contribution.currency
6562         $from
6563         $where AND civicrm_contribution.contribution_status_id = 1
6564       GROUP BY civicrm_contribution.id
6565     ) as conts
6566     GROUP BY currency";
6567
6568     $dao = CRM_Core_DAO::executeQuery($query);
6569
6570     while ($dao->fetch()) {
6571       $summary['total']['count'] += $dao->total_count;
6572       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6573       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6574     }
6575
6576     if (!empty($summary['total']['amount'])) {
6577       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6578       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6579     }
6580     else {
6581       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6582     }
6583     return $summary;
6584   }
6585
6586   /**
6587    * Add basic soft credit statistics to summary array.
6588    *
6589    * @param array $summary
6590    * @param string $where
6591    * @param string $from
6592    *
6593    * @throws \CRM_Core_Exception
6594    */
6595   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6596     $query = "
6597       SELECT COUNT( conts.total_amount ) as total_count,
6598       SUM(   conts.total_amount ) as total_amount,
6599       AVG(   conts.total_amount ) as total_avg,
6600       conts.currency as currency
6601       FROM (
6602         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6603         $from
6604         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6605         GROUP BY civicrm_contribution_soft.id
6606       ) as conts
6607       GROUP BY currency";
6608
6609     $dao = CRM_Core_DAO::executeQuery($query);
6610     $summary['soft_credit']['count'] = 0;
6611     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6612     while ($dao->fetch()) {
6613       $summary['soft_credit']['count'] += $dao->total_count;
6614       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6615       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6616     }
6617     if (!empty($summary['soft_credit']['amount'])) {
6618       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6619       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6620     }
6621     else {
6622       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6623     }
6624   }
6625
6626   /**
6627    * Add basic stats about cancelled contributions to the summary.
6628    *
6629    * @param array $summary
6630    * @param string $where
6631    * @param string $from
6632    *
6633    * @throws \CRM_Core_Exception
6634    */
6635   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6636     $query = "
6637       SELECT COUNT( conts.total_amount ) as cancel_count,
6638        SUM(   conts.total_amount ) as cancel_amount,
6639        AVG(   conts.total_amount ) as cancel_avg,
6640        conts.currency              as currency
6641         FROM (
6642       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6643       $from
6644       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6645       GROUP BY civicrm_contribution.id
6646     ) as conts
6647     GROUP BY currency";
6648
6649     $dao = CRM_Core_DAO::executeQuery($query);
6650
6651     if ($dao->N <= 1) {
6652       if ($dao->fetch()) {
6653         $summary['cancel']['count'] = $dao->cancel_count;
6654         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6655         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6656       }
6657     }
6658     else {
6659       $summary['cancel']['count'] = 0;
6660       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6661       while ($dao->fetch()) {
6662         $summary['cancel']['count'] += $dao->cancel_count;
6663         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6664         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6665       }
6666       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6667       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6668     }
6669   }
6670
6671   /**
6672    * Create the sql query for an contact search.
6673    *
6674    * @param int $offset
6675    *   The offset for the query.
6676    * @param int $rowCount
6677    *   The number of rows to return.
6678    * @param string|CRM_Utils_Sort $sort
6679    *   The order by string.
6680    * @param bool $count
6681    *   Is this a count only query ?.
6682    * @param bool $includeContactIds
6683    *   Should we include contact ids?.
6684    * @param bool $sortByChar
6685    *   If true returns the distinct array of first characters for search results.
6686    * @param bool $groupContacts
6687    *   If true, return only the contact ids.
6688    * @param string $additionalWhereClause
6689    *   If the caller wants to further restrict the search (used for components).
6690    * @param null $sortOrder
6691    * @param string $additionalFromClause
6692    *   Should be clause with proper joins, effective to reduce where clause load.
6693    *
6694    * @param bool $skipOrderAndLimit
6695    *
6696    * @return string
6697    *
6698    * @throws \CRM_Core_Exception
6699    */
6700   public function getSearchSQL(
6701     $offset = 0, $rowCount = 0, $sort = NULL,
6702     $count = FALSE, $includeContactIds = FALSE,
6703     $sortByChar = FALSE, $groupContacts = FALSE,
6704     $additionalWhereClause = NULL, $sortOrder = NULL,
6705     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6706
6707     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6708
6709     if ($sortByChar) {
6710       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6711       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6712     }
6713
6714     if ($skipOrderAndLimit) {
6715       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6716       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6717     }
6718     else {
6719       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6720     }
6721     return $query;
6722   }
6723
6724   /**
6725    * Get the component parts of the search query as an array.
6726    *
6727    * @param int $offset
6728    *   The offset for the query.
6729    * @param int $rowCount
6730    *   The number of rows to return.
6731    * @param string|CRM_Utils_Sort $sort
6732    *   The order by string.
6733    * @param bool $count
6734    *   Is this a count only query ?.
6735    * @param bool $includeContactIds
6736    *   Should we include contact ids?.
6737    * @param bool $sortByChar
6738    *   If true returns the distinct array of first characters for search results.
6739    * @param bool $groupContacts
6740    *   If true, return only the contact ids.
6741    * @param string $additionalWhereClause
6742    *   If the caller wants to further restrict the search (used for components).
6743    * @param null $sortOrder
6744    * @param string $additionalFromClause
6745    *   Should be clause with proper joins, effective to reduce where clause load.
6746    *
6747    * @return array
6748    * @throws \CRM_Core_Exception
6749    */
6750   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6751     $count = FALSE, $includeContactIds = FALSE,
6752     $sortByChar = FALSE, $groupContacts = FALSE,
6753     $additionalWhereClause = NULL, $sortOrder = NULL,
6754     $additionalFromClause = NULL) {
6755     if ($includeContactIds) {
6756       $this->_includeContactIds = TRUE;
6757       $this->_whereClause = $this->whereClause();
6758     }
6759     $onlyDeleted = in_array([
6760       'deleted_contacts',
6761       '=',
6762       '1',
6763       '0',
6764       '0',
6765     ], $this->_params);
6766
6767     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6768     // and that contact happens to be deleted, set $onlyDeleted to true
6769     foreach ($this->_params as $values) {
6770       $name = $values[0] ?? NULL;
6771       $op = $values[1] ?? NULL;
6772       $value = $values[2] ?? NULL;
6773       if ($name === 'contact_id' and $op === '=') {
6774         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6775           $onlyDeleted = TRUE;
6776         }
6777         break;
6778       }
6779     }
6780
6781     // building the query string
6782     $groupBy = $groupByCols = NULL;
6783     if (!$count) {
6784       if (isset($this->_groupByComponentClause)) {
6785         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6786         $groupByCols = explode(', ', $groupByCols);
6787       }
6788       elseif ($this->_useGroupBy) {
6789         $groupByCols = ['contact_a.id'];
6790       }
6791     }
6792     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6793       $groupByCols = ['civicrm_activity.id'];
6794     }
6795     if (!empty($groupByCols)) {
6796       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6797     }
6798
6799     $order = $orderBy = '';
6800     if (!$count) {
6801       if (!$sortByChar) {
6802         $order = $this->prepareOrderBy($sort, $sortOrder);
6803       }
6804     }
6805     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6806     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6807     //     clause. This will impact the search query output.
6808     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6809
6810     if ($disableFullGroupByMode) {
6811       CRM_Core_DAO::disableFullGroupByMode();
6812     }
6813
6814     // CRM-15231
6815     $this->_sort = $sort;
6816
6817     //CRM-15967
6818     $this->includePseudoFieldsJoin($sort);
6819
6820     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6821
6822     if ($additionalWhereClause) {
6823       $where = $where . ' AND ' . $additionalWhereClause;
6824     }
6825
6826     //additional from clause should be w/ proper joins.
6827     if ($additionalFromClause) {
6828       $from .= "\n" . $additionalFromClause;
6829     }
6830
6831     // if we are doing a transform, do it here
6832     // use the $from, $where and $having to get the contact ID
6833     if ($this->_displayRelationshipType) {
6834       $this->filterRelatedContacts($from, $where, $having);
6835     }
6836     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6837
6838     return [
6839       'select' => $select,
6840       'from' => $from,
6841       'where' => $where,
6842       'order_by' => $order,
6843       'group_by' => $groupBy,
6844       'having' => $having,
6845       'limit' => $limit,
6846     ];
6847   }
6848
6849   /**
6850    * Get the metadata for a given field.
6851    *
6852    * @param string $fieldName
6853    *
6854    * @return array
6855    */
6856   protected function getMetadataForField($fieldName) {
6857     if ($fieldName === 'contact_a.id') {
6858       // This seems to be the only anomaly.
6859       $fieldName = 'id';
6860     }
6861     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6862     $field = $this->_fields[$fieldName] ?? $pseudoField;
6863     $field = array_merge($field, $pseudoField);
6864     if (!empty($field) && empty($field['name'])) {
6865       // standardising field formatting here - over time we can phase out variants.
6866       // all paths using this currently unit tested
6867       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6868     }
6869     return $field;
6870   }
6871
6872   /**
6873    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6874    *
6875    * @param string $fieldName
6876    *
6877    * @return array
6878    */
6879   public function getMetadataForRealField($fieldName) {
6880     $field = $this->getMetadataForField($fieldName);
6881     if (!empty($field['is_pseudofield_for'])) {
6882       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6883       $field['pseudofield_name'] = $fieldName;
6884     }
6885     elseif (!empty($field['pseudoconstant'])) {
6886       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6887         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6888         if (empty($field['table_name'])) {
6889           if (!empty($field['where'])) {
6890             $field['table_name'] = explode('.', $field['where'])[0];
6891           }
6892           else {
6893             $field['table_name'] = 'civicrm_contact';
6894           }
6895         }
6896       }
6897     }
6898     return $field;
6899   }
6900
6901   /**
6902    * If we have a field that is better rendered via the pseudoconstant handled them here.
6903    *
6904    * Rather than joining in the additional table we render the option value on output.
6905    *
6906    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6907    * carefully with test coverage but aim to extend.
6908    *
6909    * @param string $name
6910    */
6911   protected function addPseudoconstantFieldToSelect($name) {
6912     $field = $this->getMetadataForRealField($name);
6913     $realFieldName = $field['name'];
6914     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6915     if ($pseudoFieldName) {
6916       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6917       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6918         'pseudoField' => $pseudoFieldName,
6919         'idCol' => $realFieldName,
6920         'field_name' => $field['name'],
6921         'bao' => $field['bao'],
6922         'pseudoconstant' => $field['pseudoconstant'],
6923       ];
6924     }
6925
6926     $this->_tables[$field['table_name']] = 1;
6927     $this->_element[$realFieldName] = 1;
6928     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6929   }
6930
6931   /**
6932    * Is this pseudofield a foreign key constraint.
6933    *
6934    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
6935    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
6936    * array which is done for contributions which are mostly handled as pseudoconstants.
6937    *
6938    * @param $fieldSpec
6939    *
6940    * @return bool
6941    */
6942   protected function isPseudoFieldAnFK($fieldSpec) {
6943     if (empty($fieldSpec['FKClassName'])
6944       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
6945       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
6946       return FALSE;
6947     }
6948     return TRUE;
6949   }
6950
6951   /**
6952    * Is the field a relative date field.
6953    *
6954    * @param string $fieldName
6955    *
6956    * @return bool
6957    */
6958   protected function isARelativeDateField($fieldName) {
6959     if (substr($fieldName, -9, 9) !== '_relative') {
6960       return FALSE;
6961     }
6962     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
6963     return isset($this->_fields[$realField]);
6964   }
6965
6966   /**
6967    * Get the specifications for the field, if available.
6968    *
6969    * @param string $fieldName
6970    *   Fieldname as displayed on the form.
6971    *
6972    * @return array
6973    */
6974   public function getFieldSpec($fieldName) {
6975     if (isset($this->_fields[$fieldName])) {
6976       $fieldSpec = $this->_fields[$fieldName];
6977       if (!empty($fieldSpec['is_pseudofield_for'])) {
6978         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
6979       }
6980       return $fieldSpec;
6981     }
6982     $lowFieldName = str_replace('_low', '', $fieldName);
6983     if (isset($this->_fields[$lowFieldName])) {
6984       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
6985     }
6986     $highFieldName = str_replace('_high', '', $fieldName);
6987     if (isset($this->_fields[$highFieldName])) {
6988       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
6989     }
6990     return [];
6991   }
6992
6993   public function buildWhereForDate() {
6994
6995   }
6996
6997   /**
6998    * Is the field a relative date field.
6999    *
7000    * @param string $fieldName
7001    *
7002    * @return bool
7003    */
7004   protected function isADateRangeField($fieldName) {
7005     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7006       return FALSE;
7007     }
7008     return !empty($this->getFieldSpec($fieldName));
7009   }
7010
7011   /**
7012    * @param $values
7013    */
7014   protected function buildRelativeDateQuery(&$values) {
7015     $value = $values[2] ?? NULL;
7016     if (empty($value)) {
7017       return;
7018     }
7019     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7020     $fieldSpec = $this->_fields[$fieldName];
7021     $tableName = $fieldSpec['table_name'];
7022     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7023     $grouping = $values[3] ?? NULL;
7024     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7025     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7026     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7027
7028     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7029     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7030     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7031
7032     $where = $fieldSpec['where'];
7033     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7034       // Special handling for contact table as it has a known alias in advanced search.
7035       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7036       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7037     }
7038
7039     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7040     if ($fieldName === 'relation_active_period_date') {
7041       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7042       // Not sure the  'right' fix.
7043       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7044       return;
7045     }
7046
7047     if (empty($dates[0])) {
7048       // ie. no start date we only have end date
7049       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7050     }
7051     elseif (empty($dates[1])) {
7052
7053       // ie. no end date we only have start date
7054       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7055     }
7056     else {
7057       // we have start and end dates.
7058       if ($secondWhere !== $where) {
7059         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7060       }
7061       else {
7062         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7063       }
7064     }
7065   }
7066
7067   /**
7068    * Build the query for a date field if it is a _high or _low field.
7069    *
7070    * @param $values
7071    *
7072    * @return bool
7073    * @throws \CRM_Core_Exception
7074    */
7075   public function buildDateRangeQuery($values) {
7076     if ($this->isADateRangeField($values[0])) {
7077       $fieldSpec = $this->getFieldSpec($values[0]);
7078       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7079       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7080       return TRUE;
7081     }
7082     return FALSE;
7083   }
7084
7085   /**
7086    * Add the address table into the query.
7087    *
7088    * @param string $tableKey
7089    * @param string $joinCondition
7090    *
7091    * @return array
7092    *   - alias name
7093    *   - address join.
7094    */
7095   protected function addAddressTable($tableKey, $joinCondition) {
7096     $tName = "$tableKey-address";
7097     $aName = "`$tableKey-address`";
7098     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7099     $this->_element["{$tName}_id"] = 1;
7100     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7101     $this->_tables[$tName] = $addressJoin;
7102
7103     return [
7104       $aName,
7105       $addressJoin,
7106     ];
7107   }
7108
7109   /**
7110    * Get the clause for group status.
7111    *
7112    * @param int $grouping
7113    *
7114    * @return string
7115    */
7116   protected function getGroupStatusClause($grouping) {
7117     $statuses = $this->getSelectedGroupStatuses($grouping);
7118     return "status IN (" . implode(', ', $statuses) . ")";
7119   }
7120
7121   /**
7122    * Get an array of the statuses that have been selected.
7123    *
7124    * @param string $grouping
7125    *
7126    * @return array
7127    *
7128    * @throws \CRM_Core_Exception
7129    */
7130   protected function getSelectedGroupStatuses($grouping) {
7131     $statuses = [];
7132     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7133     if ($gcsValues &&
7134       is_array($gcsValues[2])
7135     ) {
7136       foreach ($gcsValues[2] as $k => $v) {
7137         if ($v) {
7138           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7139         }
7140       }
7141     }
7142     else {
7143       $statuses[] = "'Added'";
7144     }
7145     return $statuses;
7146   }
7147
7148   /**
7149    * Get the qill value for the field.
7150    *
7151    * @param string $name
7152    * @param array|int|string $value
7153    * @param string $op
7154    * @param array $fieldSpec
7155    * @param string $labelOverride
7156    *   Label override, if required.
7157    *
7158    * @return string
7159    */
7160   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7161     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7162     return (string) ts("%1 %2 %3", [
7163       1 => $labelOverride ?? $fieldSpec['title'],
7164       2 => $qillop,
7165       3 => $qillVal,
7166     ]);
7167   }
7168
7169   /**
7170    * Where handling for any field with adequately defined metadata.
7171    *
7172    * @param array $fieldSpec
7173    * @param string $name
7174    * @param string|array|int $value
7175    * @param string $op
7176    * @param string|int $grouping
7177    *
7178    * @throws \CRM_Core_Exception
7179    */
7180   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7181     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7182     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7183     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7184       $this->_tables[$fieldSpec['table_name']] = 1;
7185     }
7186     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7187       $this->_whereTables[$fieldSpec['table_name']] = 1;
7188     }
7189   }
7190
7191   /**
7192    * Get the qill for the relative date range.
7193    *
7194    * @param string|null $from
7195    * @param string|null $to
7196    * @param string $fieldTitle
7197    * @param string $relativeRange
7198    *
7199    * @return string
7200    */
7201   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7202     if (!$from) {
7203       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7204     }
7205     if (!$to) {
7206       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7207     }
7208     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7209       CRM_Utils_Date::customFormat($from),
7210       CRM_Utils_Date::customFormat($to),
7211     ]) . ')';
7212   }
7213
7214 }