CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * Are we in search mode.
 218    *
 219    * @var bool
 220    */
 221   public $_search = TRUE;
 222
 223   /**
 224    * Should we skip permission checking.
 225    *
 226    * @var bool
 227    */
 228   public $_skipPermission = FALSE;
 229
 230   /**
 231    * Should we skip adding of delete clause.
 232    *
 233    * @var bool
 234    */
 235   public $_skipDeleteClause = FALSE;
 236
 237   /**
 238    * Are we in strict mode (use equality over LIKE)
 239    *
 240    * @var bool
 241    */
 242   public $_strict = FALSE;
 243
 244   /**
 245    * What operator to use to group the clauses.
 246    *
 247    * @var string
 248    */
 249   public $_operator = 'AND';
 250
 251   public $_mode = 1;
 252
 253   /**
 254    * Should we only search on primary location.
 255    *
 256    * @var bool
 257    */
 258   public $_primaryLocation = TRUE;
 259
 260   /**
 261    * Are contact ids part of the query.
 262    *
 263    * @var bool
 264    */
 265   public $_includeContactIds = FALSE;
 266
 267   /**
 268    * Should we use the smart group cache.
 269    *
 270    * @var bool
 271    */
 272   public $_smartGroupCache = TRUE;
 273
 274   /**
 275    * Should we display contacts with a specific relationship type.
 276    *
 277    * @var string
 278    */
 279   public $_displayRelationshipType;
 280
 281   /**
 282    * Reference to the query object for custom values.
 283    *
 284    * @var Object
 285    */
 286   public $_customQuery;
 287
 288   /**
 289    * Should we enable the distinct clause, used if we are including
 290    * more than one group
 291    *
 292    * @var bool
 293    */
 294   public $_useDistinct = FALSE;
 295
 296   /**
 297    * Should we just display one contact record
 298    * @var bool
 299    */
 300   public $_useGroupBy = FALSE;
 301
 302   /**
 303    * The relationship type direction
 304    *
 305    * @var array
 306    */
 307   public static $_relType;
 308
 309   /**
 310    * The activity role
 311    *
 312    * @var array
 313    */
 314   public static $_activityRole;
 315
 316   /**
 317    * Consider the component activity type
 318    * during activity search.
 319    *
 320    * @var array
 321    */
 322   public static $_considerCompActivities;
 323
 324   /**
 325    * Consider with contact activities only,
 326    * during activity search.
 327    *
 328    * @var array
 329    */
 330   public static $_withContactActivitiesOnly;
 331
 332   /**
 333    * Use distinct component clause for component searches
 334    *
 335    * @var string
 336    */
 337   public $_distinctComponentClause;
 338
 339   public $_rowCountClause;
 340
 341   /**
 342    * Use groupBy component clause for component searches
 343    *
 344    * @var string
 345    */
 346   public $_groupByComponentClause;
 347
 348   /**
 349    * Track open panes, useful in advance search
 350    *
 351    * @var array
 352    */
 353   public static $_openedPanes = [];
 354
 355   /**
 356    * For search builder - which custom fields are location-dependent
 357    * @var array
 358    */
 359   public $_locationSpecificCustomFields = [];
 360
 361   /**
 362    * The tables which have a dependency on location and/or address
 363    *
 364    * @var array
 365    */
 366   public static $_dependencies = [
 367     'civicrm_state_province' => 1,
 368     'civicrm_country' => 1,
 369     'civicrm_county' => 1,
 370     'civicrm_address' => 1,
 371     'civicrm_location_type' => 1,
 372   ];
 373
 374   /**
 375    * List of location specific fields.
 376    * @var array
 377    */
 378   public static $_locationSpecificFields = [
 379     'street_address',
 380     'street_number',
 381     'street_name',
 382     'street_unit',
 383     'supplemental_address_1',
 384     'supplemental_address_2',
 385     'supplemental_address_3',
 386     'city',
 387     'postal_code',
 388     'postal_code_suffix',
 389     'geo_code_1',
 390     'geo_code_2',
 391     'state_province',
 392     'country',
 393     'county',
 394     'phone',
 395     'email',
 396     'im',
 397     'address_name',
 398     'master_id',
 399     'location_type',
 400   ];
 401
 402   /**
 403    * Remember if we handle either end of a number or date range
 404    * so we can skip the other
 405    * @var array
 406    */
 407   protected $_rangeCache = [];
 408   /**
 409    * Set to true when $this->relationship is run to avoid adding twice.
 410    *
 411    * @var bool
 412    */
 413   protected $_relationshipValuesAdded = FALSE;
 414
 415   /**
 416    * Set to the name of the temp table if one has been created.
 417    *
 418    * @var string
 419    */
 420   public static $_relationshipTempTable;
 421
 422   public $_pseudoConstantsSelect = [];
 423
 424   public $_groupUniqueKey;
 425   public $_groupKeys = [];
 426
 427   /**
 428    * Class constructor which also does all the work.
 429    *
 430    * @param array $params
 431    * @param array $returnProperties
 432    * @param array $fields
 433    * @param bool $includeContactIds
 434    * @param bool $strict
 435    * @param bool|int $mode - mode the search is operating on
 436    *
 437    * @param bool $skipPermission
 438    * @param bool $searchDescendentGroups
 439    * @param bool $smartGroupCache
 440    * @param null $displayRelationshipType
 441    * @param string $operator
 442    * @param string $apiEntity
 443    * @param bool|null $primaryLocationOnly
 444    *
 445    * @throws \CRM_Core_Exception
 446    */
 447   public function __construct(
 448     $params = NULL, $returnProperties = NULL, $fields = NULL,
 449     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 450     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 451     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 452     $operator = 'AND',
 453     $apiEntity = NULL,
 454     $primaryLocationOnly = NULL
 455   ) {
 456     if ($primaryLocationOnly === NULL) {
 457       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 458     }
 459     $this->_primaryLocation = $primaryLocationOnly;
 460     $this->_params = &$params;
 461     if ($this->_params == NULL) {
 462       $this->_params = [];
 463     }
 464
 465     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 466       $this->_returnProperties = [];
 467     }
 468     elseif (empty($returnProperties)) {
 469       $this->_returnProperties = self::defaultReturnProperties($mode);
 470     }
 471     else {
 472       $this->_returnProperties = &$returnProperties;
 473     }
 474
 475     $this->_includeContactIds = $includeContactIds;
 476     $this->_strict = $strict;
 477     $this->_mode = $mode;
 478     $this->_skipPermission = $skipPermission;
 479     $this->_smartGroupCache = $smartGroupCache;
 480     $this->_displayRelationshipType = $displayRelationshipType;
 481     $this->setOperator($operator);
 482
 483     if ($fields) {
 484       $this->_fields = &$fields;
 485       $this->_search = FALSE;
 486       $this->_skipPermission = TRUE;
 487     }
 488     else {
 489       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 490       //  The legacy hacked fields will output as a string rather than their underlying type.
 491       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 492         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 493       }
 494       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 495       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 496       // No good option other than hard-coding metadata for this 'special' field in.
 497       $relationFields['relation_active_period_date'] = [
 498         'name' => 'relation_active_period_date',
 499         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 500         'title' => ts('Active Period'),
 501         'table_name' => 'civicrm_relationship',
 502         'where' => 'civicrm_relationship.start_date',
 503         'where_end' => 'civicrm_relationship.end_date',
 504         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 505       ];
 506       $this->_fields = array_merge($relationFields, $this->_fields);
 507
 508       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 509       unset($fields['note']);
 510       $this->_fields = array_merge($this->_fields, $fields);
 511
 512       // add activity fields
 513       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 514       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields('Case'));
 515       // Add hack as no unique name is defined for the field but the search form is in denial.
 516       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 517
 518       // add any fields provided by hook implementers
 519       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 520       $this->_fields = array_merge($this->_fields, $extFields);
 521     }
 522
 523     // basically do all the work once, and then reuse it
 524     $this->initialize($apiEntity);
 525   }
 526
 527   /**
 528    * Function which actually does all the work for the constructor.
 529    *
 530    * @param string $apiEntity
 531    *   The api entity being called.
 532    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 533    *
 534    * @throws \CRM_Core_Exception
 535    */
 536   public function initialize($apiEntity = NULL) {
 537     $this->_select = [];
 538     $this->_element = [];
 539     $this->_tables = [];
 540     $this->_whereTables = [];
 541     $this->_where = [];
 542     $this->_qill = [];
 543     $this->_cfIDs = [];
 544     $this->_paramLookup = [];
 545     $this->_having = [];
 546
 547     $this->_customQuery = NULL;
 548
 549     // reset cached static variables - CRM-5803
 550     self::$_activityRole = NULL;
 551     self::$_considerCompActivities = NULL;
 552     self::$_withContactActivitiesOnly = NULL;
 553
 554     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 555     $this->_element['contact_id'] = 1;
 556     $this->_tables['civicrm_contact'] = 1;
 557
 558     if (!empty($this->_params)) {
 559       $this->buildParamsLookup();
 560     }
 561
 562     $this->_whereTables = $this->_tables;
 563
 564     $this->selectClause($apiEntity);
 565     if (!empty($this->_cfIDs)) {
 566       // @todo This function is the select function but instead of running 'select' it
 567       // is running the whole query.
 568       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 569       $this->_customQuery->query();
 570       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 571       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 572       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 573     }
 574     $isForcePrimaryOnly = !empty($apiEntity);
 575     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 576     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 577       $component = 'contribution';
 578     }
 579     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 580       $component = 'membership';
 581     }
 582     if (isset($component) && !$this->_skipPermission) {
 583       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 584       CRM_Financial_BAO_FinancialType::buildPermissionedClause($this->_whereClause, $component);
 585     }
 586
 587     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 588     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 589
 590     $this->openedSearchPanes(TRUE);
 591   }
 592
 593   /**
 594    * Function for same purpose as convertFormValues.
 595    *
 596    * Like convert form values this function exists to pre-Process parameters from the form.
 597    *
 598    * It is unclear why they are different functions & likely relates to advances search
 599    * versus search builder.
 600    *
 601    * The direction we are going is having the form convert values to a standardised format &
 602    * moving away from weird & wonderful where clause switches.
 603    *
 604    * Fix and handle contact deletion nicely.
 605    *
 606    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 607    *
 608    * @see https://issues.civicrm.org/jira/browse/CRM-11971
 609    */
 610   public function buildParamsLookup() {
 611     $trashParamExists = FALSE;
 612     $paramByGroup = [];
 613     foreach ($this->_params as $k => $param) {
 614       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 615         $trashParamExists = TRUE;
 616       }
 617       if (!empty($param[3])) {
 618         $paramByGroup[$param[3]][$k] = $param;
 619       }
 620     }
 621
 622     if ($trashParamExists) {
 623       $this->_skipDeleteClause = TRUE;
 624
 625       //cycle through group sets and explicitly add trash param if not set
 626       foreach ($paramByGroup as $setID => $set) {
 627         if (
 628           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 629           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 630         ) {
 631           $this->_params[] = [
 632             'contact_is_deleted',
 633             '=',
 634             '0',
 635             $setID,
 636             '0',
 637           ];
 638         }
 639       }
 640     }
 641
 642     foreach ($this->_params as $value) {
 643       if (empty($value[0])) {
 644         continue;
 645       }
 646       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 647       if ($cfID) {
 648         if (!array_key_exists($cfID, $this->_cfIDs)) {
 649           $this->_cfIDs[$cfID] = [];
 650         }
 651         // Set wildcard value based on "and/or" selection
 652         foreach ($this->_params as $key => $param) {
 653           if ($param[0] == $value[0] . '_operator') {
 654             $value[4] = $param[2] == 'or';
 655             break;
 656           }
 657         }
 658         $this->_cfIDs[$cfID][] = $value;
 659       }
 660
 661       if (!array_key_exists($value[0], $this->_paramLookup)) {
 662         $this->_paramLookup[$value[0]] = [];
 663       }
 664       if ($value[0] !== 'group') {
 665         // Just trying to unravel how group interacts here! This whole function is weird.
 666         $this->_paramLookup[$value[0]][] = $value;
 667       }
 668     }
 669   }
 670
 671   /**
 672    * Some composite fields do not appear in the fields array hack to make them part of the query.
 673    *
 674    * @param $apiEntity
 675    *   The api entity being called.
 676    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 677    */
 678   public function addSpecialFields($apiEntity) {
 679     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 680     // if get called via Contact.get API having address_id as return parameter
 681     if ($apiEntity === 'Contact') {
 682       $special[] = 'address_id';
 683     }
 684     foreach ($special as $name) {
 685       if (!empty($this->_returnProperties[$name])) {
 686         if ($name === 'address_id') {
 687           $this->_tables['civicrm_address'] = 1;
 688           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 689           $this->_element['address_id'] = 1;
 690         }
 691         else {
 692           $this->_select[$name] = "contact_a.{$name} as $name";
 693           $this->_element[$name] = 1;
 694         }
 695       }
 696     }
 697   }
 698
 699   /**
 700    * Given a list of conditions in params and a list of desired
 701    * return Properties generate the required select and from
 702    * clauses. Note that since the where clause introduces new
 703    * tables, the initial attempt also retrieves all variables used
 704    * in the params list
 705    *
 706    * @param string $apiEntity
 707    *   The api entity being called.
 708    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 709    */
 710   public function selectClause($apiEntity = NULL) {
 711
 712     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 713     // param ($apiEntity) instead - presumably an oversight & 2 because
 714     // contact is not implemented as a component.
 715     $this->addSpecialFields($apiEntity);
 716
 717     foreach ($this->_fields as $name => $field) {
 718       // skip component fields
 719       // there are done by the alter query below
 720       // and need not be done on every field
 721       // @todo remove these & handle using metadata - only obscure fields
 722       // that are hack-added should need to be excluded from the main loop.
 723       if (
 724         (substr($name, 0, 12) === 'participant_') ||
 725         (substr($name, 0, 7) === 'pledge_') ||
 726         (substr($name, 0, 5) === 'case_')
 727       ) {
 728         continue;
 729       }
 730
 731       // redirect to activity select clause
 732       if (
 733         (substr($name, 0, 9) === 'activity_') ||
 734         ($name === 'parent_id')
 735       ) {
 736         CRM_Activity_BAO_Query::select($this);
 737       }
 738
 739       // if this is a hierarchical name, we ignore it
 740       $names = explode('-', $name);
 741       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 742         continue;
 743       }
 744
 745       // make an exception for special cases, to add the field in select clause
 746       $makeException = FALSE;
 747
 748       //special handling for groups/tags
 749       if (in_array($name, ['groups', 'tags', 'notes'])
 750         && isset($this->_returnProperties[substr($name, 0, -1)])
 751       ) {
 752         // @todo instead of setting make exception to get us into
 753         // an if clause that has handling for these fields buried with in it
 754         // move the handling to here.
 755         $makeException = TRUE;
 756       }
 757
 758       // since note has 3 different options we need special handling
 759       // note / note_subject / note_body
 760       if ($name === 'notes') {
 761         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 762           if (isset($this->_returnProperties[$noteField])) {
 763             $makeException = TRUE;
 764             break;
 765           }
 766         }
 767       }
 768
 769       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 770       if (
 771         !empty($this->_paramLookup[$name])
 772         || !empty($this->_returnProperties[$name])
 773         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 774         || $makeException
 775       ) {
 776         if ($cfID) {
 777           // add to cfIDs array if not present
 778           if (!array_key_exists($cfID, $this->_cfIDs)) {
 779             $this->_cfIDs[$cfID] = [];
 780           }
 781         }
 782         elseif (isset($field['where'])) {
 783           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 784           if (isset($tableName)) {
 785             if (!empty(self::$_dependencies[$tableName])) {
 786               $this->_tables['civicrm_address'] = 1;
 787               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 788               $this->_element['address_id'] = 1;
 789             }
 790
 791             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 792               $tableName === 'postal_greeting' || $tableName === 'addressee'
 793             ) {
 794               if ($tableName === 'im_provider') {
 795                 CRM_Core_OptionValue::select($this);
 796               }
 797
 798               if (in_array($tableName,
 799                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 800                 $this->_element["{$name}_id"] = 1;
 801                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 802                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 803                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 804                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 805
 806                 if ($tableName === 'email_greeting') {
 807                   // @todo bad join.
 808                   $this->_pseudoConstantsSelect[$name]['join']
 809                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 810                   $this->_pseudoConstantsSelect[$name]['join'] .=
 811                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 812                 }
 813                 elseif ($tableName === 'postal_greeting') {
 814                   // @todo bad join.
 815                   $this->_pseudoConstantsSelect[$name]['join']
 816                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 817                   $this->_pseudoConstantsSelect[$name]['join'] .=
 818                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 819                 }
 820                 elseif ($tableName == 'addressee') {
 821                   // @todo bad join.
 822                   $this->_pseudoConstantsSelect[$name]['join']
 823                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 824                   $this->_pseudoConstantsSelect[$name]['join'] .=
 825                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 826                 }
 827                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 828
 829                 //get display
 830                 $greetField = "{$name}_display";
 831                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 832                 $this->_element[$greetField] = 1;
 833                 //get custom
 834                 $greetField = "{$name}_custom";
 835                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 836                 $this->_element[$greetField] = 1;
 837               }
 838             }
 839             else {
 840               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 841                 $this->_tables[$tableName] = 1;
 842               }
 843
 844               // also get the id of the tableName
 845               $tName = substr($tableName, 8);
 846               if (in_array($tName, ['country', 'state_province', 'county'])) {
 847                 if ($tName == 'state_province') {
 848                   $this->_pseudoConstantsSelect['state_province_name'] = [
 849                     'pseudoField' => "{$tName}",
 850                     'idCol' => "{$tName}_id",
 851                     'bao' => 'CRM_Core_BAO_Address',
 852                     'table' => "civicrm_{$tName}",
 853                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 854                   ];
 855
 856                   $this->_pseudoConstantsSelect[$tName] = [
 857                     'pseudoField' => 'state_province_abbreviation',
 858                     'idCol' => "{$tName}_id",
 859                     'table' => "civicrm_{$tName}",
 860                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 861                   ];
 862                 }
 863                 else {
 864                   $this->_pseudoConstantsSelect[$name] = [
 865                     'pseudoField' => "{$tName}_id",
 866                     'idCol' => "{$tName}_id",
 867                     'bao' => 'CRM_Core_BAO_Address',
 868                     'table' => "civicrm_{$tName}",
 869                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 870                   ];
 871                 }
 872
 873                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 874                 $this->_element["{$tName}_id"] = 1;
 875               }
 876               elseif ($tName != 'contact') {
 877                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 878                 $this->_element["{$tName}_id"] = 1;
 879               }
 880
 881               //special case for phone
 882               if ($name == 'phone') {
 883                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 884                 $this->_element['phone_type_id'] = 1;
 885               }
 886
 887               // if IM then select provider_id also
 888               // to get "IM Service Provider" in a file to be exported, CRM-3140
 889               if ($name == 'im') {
 890                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 891                 $this->_element['provider_id'] = 1;
 892               }
 893
 894               if ($tName == 'contact' && $fieldName == 'organization_name') {
 895                 // special case, when current employer is set for Individual contact
 896                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 897               }
 898               elseif ($tName == 'contact' && $fieldName === 'id') {
 899                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 900               }
 901               elseif (in_array($tName, ['country', 'county'])) {
 902                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 903                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 904               }
 905               elseif ($tName == 'state_province') {
 906                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 907                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 908               }
 909               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 910                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 911                   $this->_select[$name] = "{$field['where']} as `$name`";
 912                 }
 913               }
 914               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 915                 $this->addPseudoconstantFieldToSelect($name);
 916               }
 917               else {
 918                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 919               }
 920               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 921                 $this->_element[$name] = 1;
 922               }
 923             }
 924           }
 925         }
 926         elseif ($name === 'tags') {
 927           //@todo move this handling outside the big IF & ditch $makeException
 928           $this->_useGroupBy = TRUE;
 929           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 930           $this->_element[$name] = 1;
 931           $this->_tables['civicrm_tag'] = 1;
 932           $this->_tables['civicrm_entity_tag'] = 1;
 933         }
 934         elseif ($name === 'groups') {
 935           //@todo move this handling outside the big IF & ditch $makeException
 936           $this->_useGroupBy = TRUE;
 937           // Duplicates will be created here but better to sort them out in php land.
 938           $this->_select[$name] = "
 939             CONCAT_WS(',',
 940             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 941             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 942           )
 943           as `groups`";
 944           $this->_element[$name] = 1;
 945           $this->_tables['civicrm_group_contact'] = 1;
 946           $this->_tables['civicrm_group_contact_cache'] = 1;
 947           $this->_pseudoConstantsSelect["{$name}"] = [
 948             'pseudoField' => "groups",
 949             'idCol' => 'groups',
 950           ];
 951         }
 952         elseif ($name === 'notes') {
 953           //@todo move this handling outside the big IF & ditch $makeException
 954           // if note field is subject then return subject else body of the note
 955           $noteColumn = 'note';
 956           if (isset($noteField) && $noteField === 'note_subject') {
 957             $noteColumn = 'subject';
 958           }
 959
 960           $this->_useGroupBy = TRUE;
 961           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 962           $this->_element[$name] = 1;
 963           $this->_tables['civicrm_note'] = 1;
 964         }
 965         elseif ($name === 'current_employer') {
 966           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 967           $this->_element[$name] = 1;
 968         }
 969       }
 970
 971       if ($cfID && !empty($field['is_search_range'])) {
 972         // this is a custom field with range search enabled, so we better check for two/from values
 973         if (!empty($this->_paramLookup[$name . '_from'])) {
 974           if (!array_key_exists($cfID, $this->_cfIDs)) {
 975             $this->_cfIDs[$cfID] = [];
 976           }
 977           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 978             // search in the cdID array for the same grouping
 979             $fnd = FALSE;
 980             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 981               if ($c[3] == $p[3]) {
 982                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 983                 $fnd = TRUE;
 984               }
 985             }
 986             if (!$fnd) {
 987               $p[2] = ['from' => $p[2]];
 988               $this->_cfIDs[$cfID][] = $p;
 989             }
 990           }
 991         }
 992         if (!empty($this->_paramLookup[$name . '_to'])) {
 993           if (!array_key_exists($cfID, $this->_cfIDs)) {
 994             $this->_cfIDs[$cfID] = [];
 995           }
 996           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
 997             // search in the cdID array for the same grouping
 998             $fnd = FALSE;
 999             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
1000               if ($c[4] == $p[4]) {
1001                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1002                 $fnd = TRUE;
1003               }
1004             }
1005             if (!$fnd) {
1006               $p[2] = ['to' => $p[2]];
1007               $this->_cfIDs[$cfID][] = $p;
1008             }
1009           }
1010         }
1011       }
1012     }
1013
1014     // add location as hierarchical elements
1015     $this->addHierarchicalElements();
1016
1017     // add multiple field like website
1018     $this->addMultipleElements();
1019
1020     //fix for CRM-951
1021     CRM_Core_Component::alterQuery($this, 'select');
1022
1023     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1024   }
1025
1026   /**
1027    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1028    */
1029   public function addHierarchicalElements() {
1030     if (empty($this->_returnProperties['location'])) {
1031       return;
1032     }
1033     if (!is_array($this->_returnProperties['location'])) {
1034       return;
1035     }
1036
1037     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1038     $processed = [];
1039     $index = 0;
1040
1041     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1042     $addressCustomFieldIds = [];
1043
1044     foreach ($this->_returnProperties['location'] as $name => $elements) {
1045       $lCond = self::getPrimaryCondition($name);
1046       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1047
1048       if (!$lCond) {
1049         if ($locationTypeId === FALSE) {
1050           continue;
1051         }
1052         $lCond = "location_type_id = $locationTypeId";
1053         $this->_useDistinct = TRUE;
1054
1055         //commented for CRM-3256
1056         $this->_useGroupBy = TRUE;
1057       }
1058
1059       $name = str_replace(' ', '_', $name);
1060       $tName = "$name-location_type";
1061       $ltName = "`$name-location_type`";
1062       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1063       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1064       $this->_element["{$tName}_id"] = 1;
1065       $this->_element["{$tName}"] = 1;
1066
1067       $locationTypeName = $tName;
1068       $locationTypeJoin = [];
1069
1070       $addWhereCount = 0;
1071       foreach ($elements as $elementFullName => $dontCare) {
1072         $index++;
1073         $elementName = $elementCmpName = $elementFullName;
1074
1075         if (substr($elementCmpName, 0, 5) == 'phone') {
1076           $elementCmpName = 'phone';
1077         }
1078
1079         if (array_key_exists($elementCmpName, $addressCustomFields)) {
1080           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1081             $addressCustomFieldIds[$cfID][$name] = 1;
1082           }
1083         }
1084         // add address table - doesn't matter if we do it mutliple times - it's the same data
1085         // @todo ditch the double processing of addressJoin
1086         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1087           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1088         ) {
1089           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1090           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1091           $processed[$aName] = 1;
1092         }
1093
1094         $cond = $elementType = '';
1095         if (strpos($elementName, '-') !== FALSE) {
1096           // this is either phone, email or IM
1097           list($elementName, $elementType) = explode('-', $elementName);
1098
1099           if (($elementName != 'phone') && ($elementName != 'im')) {
1100             $cond = self::getPrimaryCondition($elementType);
1101           }
1102           // CRM-13011 : If location type is primary, do not restrict search to the phone
1103           // type id - we want the primary phone, regardless of what type it is.
1104           // Otherwise, restrict to the specified phone type for the given field.
1105           if ((!$cond) && ($elementName == 'phone')) {
1106             $cond = "phone_type_id = '$elementType'";
1107           }
1108           elseif ((!$cond) && ($elementName == 'im')) {
1109             // IM service provider id, CRM-3140
1110             $cond = "provider_id = '$elementType'";
1111           }
1112           $elementType = '-' . $elementType;
1113         }
1114
1115         $field = $this->_fields[$elementName] ?? NULL;
1116         if (!empty($field)) {
1117           if (isset($this->_pseudoConstantsSelect[$field['name']])) {
1118             $this->_pseudoConstantsSelect[$name . '-' . $field['name']] = $this->_pseudoConstantsSelect[$field['name']];
1119           }
1120         }
1121
1122         // hack for profile, add location id
1123         if (!$field) {
1124           if ($elementType &&
1125             // fix for CRM-882( to handle phone types )
1126             !is_numeric($elementType)
1127           ) {
1128             if (is_numeric($name)) {
1129               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1130             }
1131             else {
1132               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1133             }
1134           }
1135           elseif (is_numeric($name)) {
1136             //this for phone type to work
1137             if (in_array($elementName, ['phone', 'phone_ext'])) {
1138               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1139             }
1140             else {
1141               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1142             }
1143           }
1144           else {
1145             //this is for phone type to work for profile edit
1146             if (in_array($elementName, ['phone', 'phone_ext'])) {
1147               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1148             }
1149             else {
1150               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1151             }
1152           }
1153         }
1154
1155         // Check if there is a value, if so also add to where Clause
1156         $addWhere = FALSE;
1157         if ($this->_params) {
1158           $nm = $elementName;
1159           if (isset($locationTypeId)) {
1160             $nm .= "-$locationTypeId";
1161           }
1162           if (!is_numeric($elementType)) {
1163             $nm .= "$elementType";
1164           }
1165
1166           foreach ($this->_params as $id => $values) {
1167             if ((is_array($values) && $values[0] == $nm) ||
1168               (in_array($elementName, ['phone', 'im'])
1169                 && (strpos($values[0], $nm) !== FALSE)
1170               )
1171             ) {
1172               $addWhere = TRUE;
1173               $addWhereCount++;
1174               break;
1175             }
1176           }
1177         }
1178
1179         if ($field && isset($field['where'])) {
1180           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1181           $pf = substr($tableName, 8);
1182           $tName = $name . '-' . $pf . $elementType;
1183           if (isset($tableName)) {
1184             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1185               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1186             }
1187             else {
1188               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1189             }
1190
1191             $this->_element["{$tName}_id"] = 1;
1192             if (substr($tName, -15) == '-state_province') {
1193               // FIXME: hack to fix CRM-1900
1194               $a = Civi::settings()->get('address_format');
1195
1196               if (substr_count($a, 'state_province_name') > 0) {
1197                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1198                   'pseudoField' => "{$pf}_id",
1199                   'idCol' => "{$tName}_id",
1200                   'bao' => 'CRM_Core_BAO_Address',
1201                 ];
1202                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1203               }
1204               else {
1205                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1206                   'pseudoField' => 'state_province_abbreviation',
1207                   'idCol' => "{$tName}_id",
1208                 ];
1209                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1210               }
1211             }
1212             else {
1213               if (substr($elementFullName, 0, 2) == 'im') {
1214                 $provider = "{$name}-{$elementFullName}-provider_id";
1215                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1216                 $this->_element[$provider] = 1;
1217               }
1218               if ($pf == 'country' || $pf == 'county') {
1219                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1220                   'pseudoField' => "{$pf}_id",
1221                   'idCol' => "{$tName}_id",
1222                   'bao' => 'CRM_Core_BAO_Address',
1223                 ];
1224                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1225               }
1226               else {
1227                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1228               }
1229             }
1230
1231             if (in_array($pf, ['state_province', 'country', 'county'])) {
1232               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1233             }
1234             else {
1235               $this->_element["{$name}-{$elementFullName}"] = 1;
1236             }
1237
1238             if (empty($processed["`$tName`"])) {
1239               $processed["`$tName`"] = 1;
1240               $newName = $tableName . '_' . $index;
1241               switch ($tableName) {
1242                 case 'civicrm_phone':
1243                 case 'civicrm_email':
1244                 case 'civicrm_im':
1245                 case 'civicrm_openid':
1246
1247                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1248                   if ($tableName != 'civicrm_phone') {
1249                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1250                   }
1251                   elseif (is_numeric($name)) {
1252                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1253                   }
1254
1255                   // this special case to add phone type
1256                   if ($cond) {
1257                     $phoneTypeCondition = " AND `$tName`.$cond ";
1258                     //gross hack to pickup corrupted data also, CRM-7603
1259                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1260                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1261                       if (!empty($lCond)) {
1262                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1263                       }
1264                     }
1265                     $this->_tables[$tName] .= $phoneTypeCondition;
1266                   }
1267
1268                   //build locationType join
1269                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1270
1271                   if ($addWhere) {
1272                     $this->_whereTables[$tName] = $this->_tables[$tName];
1273                   }
1274                   break;
1275
1276                 case 'civicrm_state_province':
1277                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1278                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1279                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1280                   if ($addWhere) {
1281                     $this->_whereTables["{$name}-address"] = $addressJoin;
1282                   }
1283                   break;
1284
1285                 case 'civicrm_location_type':
1286                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.location_type_id";
1287
1288                   if ($addWhere) {
1289                     $this->_whereTables["{$name}-address"] = $addressJoin;
1290                   }
1291                   break;
1292
1293                 case 'civicrm_country':
1294                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1295                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1296                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1297                   if ($addWhere) {
1298                     $this->_whereTables["{$name}-address"] = $addressJoin;
1299                   }
1300                   break;
1301
1302                 case 'civicrm_county':
1303                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1304                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1305                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1306                   if ($addWhere) {
1307                     $this->_whereTables["{$name}-address"] = $addressJoin;
1308                   }
1309                   break;
1310
1311                 default:
1312                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1313                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1314                   }
1315                   if ($addWhere) {
1316                     $this->_whereTables["{$name}-address"] = $addressJoin;
1317                   }
1318                   break;
1319               }
1320             }
1321           }
1322         }
1323       }
1324
1325       // add location type  join
1326       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1327       $this->_tables[$locationTypeName] = $ltypeJoin;
1328
1329       // table should be present in $this->_whereTables,
1330       // to add its condition in location type join, CRM-3939.
1331       if ($addWhereCount) {
1332         $locClause = [];
1333         foreach ($this->_whereTables as $tableName => $clause) {
1334           if (!empty($locationTypeJoin[$tableName])) {
1335             $locClause[] = $locationTypeJoin[$tableName];
1336           }
1337         }
1338
1339         if (!empty($locClause)) {
1340           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1341         }
1342       }
1343     }
1344
1345     if (!empty($addressCustomFieldIds)) {
1346       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1347       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1348         foreach ($locTypeName as $name => $dnc) {
1349           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1350           $fieldName = "$name-custom_{$cfID}";
1351           $tName = "$name-address-custom-{$cfID}";
1352           $aName = "`$name-address-custom-{$cfID}`";
1353           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1354           $this->_element["{$tName}_id"] = 1;
1355           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1356           $this->_element[$fieldName] = 1;
1357           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1358         }
1359       }
1360     }
1361   }
1362
1363   /**
1364    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1365    */
1366   public function addMultipleElements() {
1367     if (empty($this->_returnProperties['website'])) {
1368       return;
1369     }
1370     if (!is_array($this->_returnProperties['website'])) {
1371       return;
1372     }
1373
1374     foreach ($this->_returnProperties['website'] as $key => $elements) {
1375       foreach ($elements as $elementFullName => $dontCare) {
1376         $tName = "website-{$key}-{$elementFullName}";
1377         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1378         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1379         $this->_element["{$tName}_id"] = 1;
1380         $this->_element["{$tName}"] = 1;
1381
1382         $type = "website-{$key}-website_type_id";
1383         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1384         $this->_element[$type] = 1;
1385         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1386       }
1387     }
1388   }
1389
1390   /**
1391    * Generate the query based on what type of query we need.
1392    *
1393    * @param bool $count
1394    * @param bool $sortByChar
1395    * @param bool $groupContacts
1396    * @param bool $onlyDeleted
1397    *
1398    * @return array
1399    *   sql query parts as an array
1400    */
1401   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1402     // build permission clause
1403     $this->generatePermissionClause($onlyDeleted, $count);
1404
1405     if ($count) {
1406       if (isset($this->_rowCountClause)) {
1407         $select = "SELECT {$this->_rowCountClause}";
1408       }
1409       elseif (isset($this->_distinctComponentClause)) {
1410         // we add distinct to get the right count for components
1411         // for the more complex result set, we use GROUP BY the same id
1412         // CRM-9630
1413         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1414       }
1415       else {
1416         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1417       }
1418       $from = $this->_simpleFromClause;
1419       if ($this->_useDistinct) {
1420         $this->_useGroupBy = TRUE;
1421       }
1422     }
1423     elseif ($sortByChar) {
1424       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1425       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1426       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1427       $from = $this->_simpleFromClause;
1428     }
1429     elseif ($groupContacts) {
1430       $select = 'SELECT contact_a.id as id';
1431       if ($this->_useDistinct) {
1432         $this->_useGroupBy = TRUE;
1433       }
1434       $from = $this->_simpleFromClause;
1435     }
1436     else {
1437       if (!empty($this->_paramLookup['group'])) {
1438
1439         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1440
1441         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1442           $this->_paramLookup['group'][0][1] = key($value);
1443         }
1444
1445         // Presumably the lines below come into manage groups screen.
1446         // make sure there is only one element
1447         // this is used when we are running under smog and need to know
1448         // how the contact was added (CRM-1203)
1449         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1450         if ((count($this->_paramLookup['group']) == 1) &&
1451           (count($groups) == 1)
1452         ) {
1453           $groupId = $groups[0];
1454
1455           //check if group is saved search
1456           $group = new CRM_Contact_BAO_Group();
1457           $group->id = $groupId;
1458           $group->find(TRUE);
1459
1460           if (!isset($group->saved_search_id)) {
1461             $tbName = "civicrm_group_contact";
1462             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1463             // as this will add load to an intentionally light query.
1464             // ideally this code would be removed as it appears to be to support CRM-1203
1465             // and passing in the required returnProperties from the url would
1466             // make more sense that globally applying the requirements of one form.
1467             if (($this->_returnProperties != ['contact_id'])) {
1468               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1469               $this->_element['group_contact_id'] = 1;
1470               $this->_select['status'] = "$tbName.status as status";
1471               $this->_element['status'] = 1;
1472             }
1473           }
1474         }
1475         $this->_useGroupBy = TRUE;
1476       }
1477       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1478         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1479           // CRM-5954
1480           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1481           $this->_useDistinct = FALSE;
1482           $this->_useGroupBy = TRUE;
1483         }
1484       }
1485
1486       $select = $this->getSelect();
1487       $from = $this->_fromClause;
1488     }
1489
1490     $where = '';
1491     if (!empty($this->_whereClause)) {
1492       $where = "WHERE {$this->_whereClause}";
1493     }
1494
1495     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1496       if (!empty($this->_permissionFromClause)) {
1497         $from .= " $this->_permissionFromClause";
1498       }
1499       if (empty($where)) {
1500         $where = "WHERE $this->_permissionWhereClause";
1501       }
1502       else {
1503         $where = "$where AND $this->_permissionWhereClause";
1504       }
1505     }
1506
1507     $having = '';
1508     if (!empty($this->_having)) {
1509       foreach ($this->_having as $havingSets) {
1510         foreach ($havingSets as $havingSet) {
1511           $havingValue[] = $havingSet;
1512         }
1513       }
1514       $having = ' HAVING ' . implode(' AND ', $havingValue);
1515     }
1516
1517     // if we are doing a transform, do it here
1518     // use the $from, $where and $having to get the contact ID
1519     if ($this->_displayRelationshipType) {
1520       $this->filterRelatedContacts($from, $where, $having);
1521     }
1522
1523     return [$select, $from, $where, $having];
1524   }
1525
1526   /**
1527    * Get where values from the parameters.
1528    *
1529    * @param string $name
1530    * @param mixed $grouping
1531    *
1532    * @return mixed
1533    */
1534   public function getWhereValues($name, $grouping) {
1535     $result = NULL;
1536     foreach ($this->_params as $values) {
1537       if ($values[0] == $name && $values[3] == $grouping) {
1538         return $values;
1539       }
1540     }
1541
1542     return $result;
1543   }
1544
1545   /**
1546    * Fix date values.
1547    *
1548    * @param bool $relative
1549    * @param string $from
1550    * @param string $to
1551    */
1552   public static function fixDateValues($relative, &$from, &$to) {
1553     if ($relative) {
1554       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1555     }
1556   }
1557
1558   /**
1559    * Convert values from form-appropriate to query-object appropriate.
1560    *
1561    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1562    * So, ideally we would convert all fields to look like
1563    *  array(
1564    *   0 => $fieldName
1565    *   // Set the operator for legacy reasons, but it is ignored
1566    *   1 =>  '='
1567    *   // array in sql filter syntax
1568    *   2 => array('BETWEEN' => array(1,60),
1569    *   3 => null
1570    *   4 => null
1571    *  );
1572    *
1573    * There are some examples of the syntax in
1574    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1575    *
1576    * More notes at CRM_Core_DAO::createSQLFilter
1577    *
1578    * and a list of supported operators in CRM_Core_DAO
1579    *
1580    * @param array $formValues
1581    * @param int $wildcard
1582    * @param bool $useEquals
1583    *
1584    * @param string $apiEntity
1585    *
1586    * @param array $entityReferenceFields
1587    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1588    *
1589    * @return array
1590    */
1591   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1592     $entityReferenceFields = []) {
1593     $params = [];
1594     if (empty($formValues)) {
1595       return $params;
1596     }
1597
1598     self::filterCountryFromValuesIfStateExists($formValues);
1599     CRM_Core_BAO_CustomValue::fixCustomFieldValue($formValues);
1600
1601     foreach ($formValues as $id => $values) {
1602       if (self::isAlreadyProcessedForQueryFormat($values)) {
1603         $params[] = $values;
1604         continue;
1605       }
1606
1607       self::legacyConvertFormValues($id, $values);
1608
1609       // The form uses 1 field to represent two db fields
1610       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1611         $contactType = [];
1612         $subType = [];
1613         foreach ((array) $values as $key => $type) {
1614           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1615           $contactType[$types[0]] = $types[0];
1616           // Add sub-type if specified
1617           if (!empty($types[1])) {
1618             $subType[$types[1]] = $types[1];
1619           }
1620         }
1621         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1622         if ($subType) {
1623           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1624         }
1625       }
1626       elseif ($id === 'privacy') {
1627         if (is_array($formValues['privacy'])) {
1628           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1629           foreach ($formValues['privacy'] as $key => $value) {
1630             if ($value) {
1631               $params[] = [$key, $op, $value, 0, 0];
1632             }
1633           }
1634         }
1635       }
1636       elseif ($id === 'email_on_hold') {
1637         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1638           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1639           // so in 5.11 we have an extra if that should become redundant over time.
1640           // https://lab.civicrm.org/dev/core/issues/745
1641           // @todo this renaming of email_on_hold to on_hold needs revisiting
1642           // it precedes recent changes but causes the default not to reload.
1643           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1644           if (!empty($onHoldValue)) {
1645             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1646           }
1647         }
1648       }
1649       elseif (substr($id, 0, 7) === 'custom_'
1650         &&  (
1651           substr($id, -5, 5) === '_from'
1652           || substr($id, -3, 3) === '_to'
1653         )
1654       ) {
1655         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1656       }
1657       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1658         FALSE)) {
1659         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1660       }
1661       else {
1662         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1663
1664         if (!$values) {
1665           continue;
1666         }
1667         $params[] = $values;
1668       }
1669     }
1670     return $params;
1671   }
1672
1673   /**
1674    * Function to support legacy format for groups and tags.
1675    *
1676    * @param string $id
1677    * @param array|int $values
1678    *
1679    */
1680   public static function legacyConvertFormValues($id, &$values) {
1681     $legacyElements = [
1682       'group',
1683       'tag',
1684       'contact_tags',
1685       'contact_type',
1686       'membership_type_id',
1687       'membership_status_id',
1688     ];
1689     if (in_array($id, $legacyElements) && is_array($values)) {
1690       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1691       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1692       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1693     }
1694   }
1695
1696   /**
1697    * Fix values from query from/to something no-one cared enough to document.
1698    *
1699    * @param int $id
1700    * @param array $values
1701    * @param int $wildcard
1702    * @param bool $useEquals
1703    *
1704    * @param string $apiEntity
1705    *
1706    * @return array|null
1707    */
1708   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1709     // skip a few search variables
1710     static $skipWhere = NULL;
1711     static $likeNames = NULL;
1712     $result = NULL;
1713
1714     // Change camelCase EntityName to lowercase with underscores
1715     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1716
1717     // check if $value is in OK (Operator as Key) format as used by Get API
1718     if (CRM_Utils_System::isNull($values)) {
1719       return $result;
1720     }
1721
1722     if (!$skipWhere) {
1723       $skipWhere = [
1724         'task',
1725         'radio_ts',
1726         'uf_group_id',
1727         'component_mode',
1728         'qfKey',
1729         'operator',
1730         'display_relationship_type',
1731       ];
1732     }
1733
1734     if (in_array($id, $skipWhere) ||
1735       substr($id, 0, 4) == '_qf_' ||
1736       substr($id, 0, 7) == 'hidden_'
1737     ) {
1738       return $result;
1739     }
1740
1741     if ($apiEntity &&
1742       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1743       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1744       (substr($id, 0, 7) != 'custom_')
1745     ) {
1746       $id = $apiEntity . '_' . $id;
1747     }
1748
1749     if (!$likeNames) {
1750       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1751     }
1752
1753     // email comes in via advanced search
1754     // so use wildcard always
1755     if ($id == 'email') {
1756       $wildcard = 1;
1757     }
1758
1759     if (!$useEquals && in_array($id, $likeNames)) {
1760       $result = [$id, 'LIKE', $values, 0, 1];
1761     }
1762     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1763       $result = [$id, 'LIKE', $values, 0, 0];
1764     }
1765     elseif ($id == 'contact_type' ||
1766       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1767     ) {
1768       $result = [$id, 'IN', $values, 0, $wildcard];
1769     }
1770     else {
1771       $result = [$id, '=', $values, 0, $wildcard];
1772     }
1773
1774     return $result;
1775   }
1776
1777   /**
1778    * Get the where clause for a single field.
1779    *
1780    * @param array $values
1781    * @param bool $isForcePrimaryOnly
1782    *
1783    * @throws \CRM_Core_Exception
1784    */
1785   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1786     if ($this->isARelativeDateField($values[0])) {
1787       $this->buildRelativeDateQuery($values);
1788       return;
1789     }
1790     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1791
1792     // do not process custom fields or prefixed contact ids or component params
1793     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1794       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1795       (substr($values[0], 0, 13) === 'contribution_') ||
1796       (substr($values[0], 0, 6) === 'event_') ||
1797       (substr($values[0], 0, 12) === 'participant_') ||
1798       (substr($values[0], 0, 7) === 'member_') ||
1799       (substr($values[0], 0, 6) === 'grant_') ||
1800       (substr($values[0], 0, 7) === 'pledge_') ||
1801       (substr($values[0], 0, 5) === 'case_') ||
1802       (substr($values[0], 0, 10) === 'financial_') ||
1803       (substr($values[0], 0, 8) === 'payment_') ||
1804       (substr($values[0], 0, 11) === 'membership_')
1805       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1806       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1807       // be needed.
1808       || $values[0] === 'mailing_id'
1809     ) {
1810       return;
1811     }
1812
1813     // skip for hook injected fields / params
1814     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1815     if (array_key_exists($values[0], $extFields)) {
1816       return;
1817     }
1818
1819     switch ($values[0]) {
1820       case 'deleted_contacts':
1821         $this->deletedContacts($values);
1822         return;
1823
1824       case 'contact_sub_type':
1825         $this->contactSubType($values);
1826         return;
1827
1828       case 'group':
1829       case 'group_type':
1830         $this->group($values);
1831         return;
1832
1833       // case tag comes from find contacts
1834       case 'tag_search':
1835         $this->tagSearch($values);
1836         return;
1837
1838       case 'tag':
1839       case 'contact_tags':
1840         $this->tag($values);
1841         return;
1842
1843       case 'note':
1844       case 'note_body':
1845       case 'note_subject':
1846         $this->notes($values);
1847         return;
1848
1849       case 'uf_user':
1850         $this->ufUser($values);
1851         return;
1852
1853       case 'sort_name':
1854       case 'display_name':
1855         $this->sortName($values);
1856         return;
1857
1858       case 'addressee':
1859       case 'postal_greeting':
1860       case 'email_greeting':
1861         $this->greetings($values);
1862         return;
1863
1864       case 'email':
1865       case 'email_id':
1866         $this->email($values, $isForcePrimaryOnly);
1867         return;
1868
1869       case 'phone_numeric':
1870         $this->phone_numeric($values);
1871         return;
1872
1873       case 'phone_phone_type_id':
1874       case 'phone_location_type_id':
1875         $this->phone_option_group($values);
1876         return;
1877
1878       case 'street_address':
1879         $this->street_address($values);
1880         return;
1881
1882       case 'street_number':
1883         $this->street_number($values);
1884         return;
1885
1886       case 'sortByCharacter':
1887         $this->sortByCharacter($values);
1888         return;
1889
1890       case 'location_type':
1891         $this->locationType($values);
1892         return;
1893
1894       case 'county':
1895         $this->county($values);
1896         return;
1897
1898       case 'state_province':
1899       case 'state_province_id':
1900       case 'state_province_name':
1901         $this->stateProvince($values);
1902         return;
1903
1904       case 'country':
1905       case 'country_id':
1906         $this->country($values, FALSE);
1907         return;
1908
1909       case 'postal_code':
1910       case 'postal_code_low':
1911       case 'postal_code_high':
1912         $this->postalCode($values);
1913         return;
1914
1915       case 'activity_date':
1916       case 'activity_date_low':
1917       case 'activity_date_high':
1918       case 'activity_date_time_low':
1919       case 'activity_date_time_high':
1920       case 'activity_role':
1921       case 'activity_status_id':
1922       case 'activity_status':
1923       case 'activity_priority':
1924       case 'activity_priority_id':
1925       case 'followup_parent_id':
1926       case 'parent_id':
1927       case 'source_contact_id':
1928       case 'activity_text':
1929       case 'activity_option':
1930       case 'test_activities':
1931       case 'activity_type_id':
1932       case 'activity_type':
1933       case 'activity_survey_id':
1934       case 'activity_tags':
1935       case 'activity_taglist':
1936       case 'activity_test':
1937       case 'activity_campaign_id':
1938       case 'activity_engagement_level':
1939       case 'activity_id':
1940       case 'activity_result':
1941       case 'source_contact':
1942         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1943         return;
1944
1945       case 'age_low':
1946       case 'age_high':
1947       case 'birth_date_low':
1948       case 'birth_date_high':
1949       case 'deceased_date_low':
1950       case 'deceased_date_high':
1951         $this->demographics($values);
1952         return;
1953
1954       case 'age_asof_date':
1955         // handled by demographics
1956         return;
1957
1958       case 'log_date_low':
1959       case 'log_date_high':
1960         $this->modifiedDates($values);
1961         return;
1962
1963       case 'changed_by':
1964         $this->changeLog($values);
1965         return;
1966
1967       case 'do_not_phone':
1968       case 'do_not_email':
1969       case 'do_not_mail':
1970       case 'do_not_sms':
1971       case 'do_not_trade':
1972       case 'is_opt_out':
1973         $this->privacy($values);
1974         return;
1975
1976       case 'privacy_options':
1977         $this->privacyOptions($values);
1978         return;
1979
1980       case 'privacy_operator':
1981       case 'privacy_toggle':
1982         // these are handled by privacy options
1983         return;
1984
1985       case 'preferred_communication_method':
1986         $this->preferredCommunication($values);
1987         return;
1988
1989       case 'relation_type_id':
1990       case 'relationship_start_date_high':
1991       case 'relationship_start_date_low':
1992       case 'relationship_end_date_high':
1993       case 'relationship_end_date_low':
1994       case 'relation_active_period_date_high':
1995       case 'relation_active_period_date_low':
1996       case 'relation_target_name':
1997       case 'relation_status':
1998       case 'relation_description':
1999       case 'relation_date_low':
2000       case 'relation_date_high':
2001         $this->relationship($values);
2002         $this->_relationshipValuesAdded = TRUE;
2003         return;
2004
2005       case 'task_status_id':
2006         $this->task($values);
2007         return;
2008
2009       case 'task_id':
2010         // since this case is handled with the above
2011         return;
2012
2013       case 'prox_distance':
2014         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2015         return;
2016
2017       case 'prox_street_address':
2018       case 'prox_city':
2019       case 'prox_postal_code':
2020       case 'prox_state_province_id':
2021       case 'prox_country_id':
2022       case 'prox_geo_code_1':
2023       case 'prox_geo_code_2':
2024         // handled by the proximity_distance clause
2025         return;
2026
2027       default:
2028         $this->restWhere($values);
2029         return;
2030     }
2031   }
2032
2033   /**
2034    * Given a list of conditions in params generate the required where clause.
2035    *
2036    * @param bool $isForcePrimaryEmailOnly
2037    *
2038    * @return string
2039    * @throws \CRM_Core_Exception
2040    */
2041   public function whereClause($isForcePrimaryEmailOnly = NULL) {
2042     $this->_where[0] = [];
2043     $this->_qill[0] = [];
2044
2045     $this->includeContactIDs();
2046     if (!empty($this->_params)) {
2047       foreach (array_keys($this->_params) as $id) {
2048         if (empty($this->_params[$id][0])) {
2049           continue;
2050         }
2051         // check for both id and contact_id
2052         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2053           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2054           $field = $this->_fields['id'] ?? NULL;
2055           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2056             'CRM_Contact_BAO_Contact',
2057             "contact_a.id",
2058             $this->_params[$id][2],
2059             $this->_params[$id][1]
2060           );
2061           $this->_qill[0][] = ts("%1 %2 %3", [
2062             1 => $field['title'] ?? '',
2063             2 => $qillop,
2064             3 => $qillVal,
2065           ]);
2066         }
2067         else {
2068           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2069         }
2070       }
2071
2072       CRM_Core_Component::alterQuery($this, 'where');
2073
2074       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2075     }
2076
2077     if ($this->_customQuery) {
2078       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2079       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2080       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2081       if (!empty($this->_customQuery->_where)) {
2082         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2083       }
2084       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2085     }
2086
2087     $clauses = [];
2088     $andClauses = [];
2089
2090     $validClauses = 0;
2091     if (!empty($this->_where)) {
2092       foreach ($this->_where as $grouping => $values) {
2093         if ($grouping > 0 && !empty($values)) {
2094           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2095           $validClauses++;
2096         }
2097       }
2098
2099       if (!empty($this->_where[0])) {
2100         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2101       }
2102       if (!empty($clauses)) {
2103         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2104       }
2105
2106       if ($validClauses > 1) {
2107         $this->_useDistinct = TRUE;
2108       }
2109     }
2110
2111     return implode(' AND ', $andClauses);
2112   }
2113
2114   /**
2115    * Generate where clause for any parameters not already handled.
2116    *
2117    * @param array $values
2118    *
2119    * @throws Exception
2120    */
2121   public function restWhere(&$values) {
2122     $name = $values[0] ?? NULL;
2123     $op = $values[1] ?? NULL;
2124     $value = $values[2] ?? NULL;
2125     $grouping = $values[3] ?? NULL;
2126     $wildcard = $values[4] ?? NULL;
2127
2128     if (isset($grouping) && empty($this->_where[$grouping])) {
2129       $this->_where[$grouping] = [];
2130     }
2131
2132     $multipleFields = ['url'];
2133
2134     //check if the location type exists for fields
2135     $lType = '';
2136     $locType = explode('-', $name);
2137
2138     if (!in_array($locType[0], $multipleFields)) {
2139       //add phone type if exists
2140       if (isset($locType[2]) && $locType[2]) {
2141         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2142       }
2143     }
2144
2145     $field = $this->_fields[$name] ?? NULL;
2146
2147     if (!$field) {
2148       $field = $this->_fields[$locType[0]] ?? NULL;
2149
2150       if (!$field) {
2151         // Strip any trailing _high & _low that might be appended.
2152         $realFieldName = str_replace(['_high', '_low'], '', $name);
2153         if (isset($this->_fields[$realFieldName])) {
2154           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2155           $columnName = $field['column_name'] ?? $field['name'];
2156           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2157         }
2158         return;
2159       }
2160     }
2161
2162     $setTables = TRUE;
2163
2164     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2165     if (isset($locType[1]) && is_numeric($locType[1])) {
2166       $lType = $locationType[$locType[1]];
2167     }
2168     if ($lType) {
2169       $field['title'] .= " ($lType)";
2170     }
2171
2172     if (substr($name, 0, 14) === 'state_province') {
2173       if (isset($locType[1]) && is_numeric($locType[1])) {
2174         $setTables = FALSE;
2175         $aName = "{$lType}-address";
2176         $where = "`$aName`.state_province_id";
2177       }
2178       else {
2179         $where = "civicrm_address.state_province_id";
2180       }
2181
2182       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2183       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2184       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2185       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2186     }
2187     elseif (!empty($field['pseudoconstant'])) {
2188       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2189       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2190       $this->optionValueQuery(
2191         $name, $op, $value, $grouping,
2192         'CRM_Contact_DAO_Contact',
2193         $field,
2194         $field['html']['label'] ?? $field['title'],
2195         CRM_Utils_Type::typeToString($dataType)
2196       );
2197       if ($name === 'gender_id') {
2198         self::$_openedPanes[ts('Demographics')] = TRUE;
2199       }
2200     }
2201     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2202       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2203       if (isset($locType[1]) && is_numeric($locType[1])) {
2204         $setTables = FALSE;
2205         $aName = "{$lType}-address";
2206         $where = "`$aName`.$name";
2207       }
2208       else {
2209         $where = "civicrm_address.$name";
2210       }
2211
2212       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2213       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2214
2215       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2216       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2217     }
2218     elseif ($name === 'world_region') {
2219       $this->optionValueQuery(
2220         $name, $op, $value, $grouping,
2221         NULL,
2222         $field,
2223         ts('World Region'),
2224         'Positive'
2225       );
2226     }
2227     elseif ($name === 'is_deceased') {
2228       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2229       self::$_openedPanes[ts('Demographics')] = TRUE;
2230     }
2231     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2232       $appendDateTime = TRUE;
2233       if ($name === 'deceased_date' || $name === 'birth_date') {
2234         $appendDateTime = FALSE;
2235         self::$_openedPanes[ts('Demographics')] = TRUE;
2236       }
2237       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2238     }
2239     elseif ($name === 'contact_id') {
2240       if (is_int($value)) {
2241         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2242         $this->_qill[$grouping][] = "$field[title] $op $value";
2243       }
2244     }
2245     elseif ($name === 'name') {
2246       $value = CRM_Core_DAO::escapeString($value);
2247       if ($wildcard) {
2248         $op = 'LIKE';
2249         $value = self::getWildCardedValue($wildcard, $op, $value);
2250       }
2251       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2252       // @todo it's likely this code path is obsolete / never called. It is definitely not
2253       // passed through in our test suite.
2254       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2255       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2256     }
2257     elseif ($name === 'current_employer') {
2258       if ($wildcard) {
2259         $op = 'LIKE';
2260         $value = self::getWildCardedValue($wildcard, $op, $value);
2261       }
2262       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2263         $value
2264       );
2265       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2266       $this->_where[$grouping][] = $ceWhereClause;
2267       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2268     }
2269     elseif (substr($name, 0, 4) === 'url-') {
2270       $tName = 'civicrm_website';
2271       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2272       $value = CRM_Core_DAO::escapeString($value);
2273       if ($wildcard) {
2274         $op = 'LIKE';
2275         $value = self::getWildCardedValue($wildcard, $op, $value);
2276       }
2277
2278       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2279       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2280     }
2281     elseif ($name === 'contact_is_deleted') {
2282       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2283     }
2284     elseif (!empty($field['where'])) {
2285       $type = NULL;
2286       if (!empty($field['type'])) {
2287         $type = CRM_Utils_Type::typeToString($field['type']);
2288       }
2289
2290       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2291
2292       if (isset($locType[1]) &&
2293         is_numeric($locType[1])
2294       ) {
2295         $setTables = FALSE;
2296
2297         //get the location name
2298         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2299         $fieldName = "`$tName`.$fldName";
2300
2301         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2302         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2303
2304       }
2305       else {
2306         if ($tableName == 'civicrm_contact') {
2307           $fieldName = "contact_a.{$fieldName}";
2308         }
2309         else {
2310           $fieldName = $field['where'];
2311         }
2312       }
2313
2314       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2315       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2316         1 => $field['title'],
2317         2 => $qillop,
2318         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2319       ]);
2320
2321       if (is_array($value)) {
2322         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2323         // for api style operators for functions that hit this point without worrying about regression
2324         // (the previous comments indicated the condition for hitting this point were unknown
2325         // per CRM-14743 we are adding modified_date & created_date operator support
2326         $operations = array_keys($value);
2327         foreach ($operations as $operator) {
2328           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2329             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2330             //so this condition will satisfy the search for now
2331             if (strpos($op, 'IN') !== FALSE) {
2332               $value = [$op => $value];
2333             }
2334             // we don't know when this might happen
2335             else {
2336               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2337             }
2338           }
2339         }
2340         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2341       }
2342       else {
2343         if ($wildcard) {
2344           $op = 'LIKE';
2345           $value = self::getWildCardedValue($wildcard, $op, $value);
2346         }
2347
2348         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2349       }
2350     }
2351
2352     if ($setTables && isset($field['where'])) {
2353       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2354       if (isset($tableName)) {
2355         $this->_tables[$tableName] = 1;
2356         $this->_whereTables[$tableName] = 1;
2357       }
2358     }
2359   }
2360
2361   /**
2362    * @param $where
2363    * @param $locType
2364    *
2365    * @return array
2366    * @throws Exception
2367    */
2368   public static function getLocationTableName(&$where, &$locType) {
2369     if (isset($locType[1]) && is_numeric($locType[1])) {
2370       list($tbName, $fldName) = explode(".", $where);
2371
2372       //get the location name
2373       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2374       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2375       if (in_array($locType[0], $specialFields)) {
2376         //hack to fix / special handing for phone_ext
2377         if ($locType[0] == 'phone_ext') {
2378           $locType[0] = 'phone';
2379         }
2380         if (isset($locType[2]) && $locType[2]) {
2381           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2382         }
2383         else {
2384           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2385         }
2386       }
2387       elseif (in_array($locType[0],
2388         [
2389           'address_name',
2390           'street_address',
2391           'street_name',
2392           'street_number_suffix',
2393           'street_unit',
2394           'supplemental_address_1',
2395           'supplemental_address_2',
2396           'supplemental_address_3',
2397           'city',
2398           'postal_code',
2399           'postal_code_suffix',
2400           'geo_code_1',
2401           'geo_code_2',
2402           'master_id',
2403         ]
2404       )) {
2405         //fix for search by profile with address fields.
2406         $tName = "{$locationType[$locType[1]]}-address";
2407       }
2408       elseif (in_array($locType[0],
2409           [
2410             'on_hold',
2411             'signature_html',
2412             'signature_text',
2413             'is_bulkmail',
2414           ]
2415         )) {
2416         $tName = "{$locationType[$locType[1]]}-email";
2417       }
2418       elseif ($locType[0] == 'provider_id') {
2419         $tName = "{$locationType[$locType[1]]}-im";
2420       }
2421       elseif ($locType[0] == 'openid') {
2422         $tName = "{$locationType[$locType[1]]}-openid";
2423       }
2424       else {
2425         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2426       }
2427       $tName = str_replace(' ', '_', $tName);
2428       return [$tName, $fldName];
2429     }
2430     throw new CRM_Core_Exception('Cannot determine location table information');
2431   }
2432
2433   /**
2434    * Given a result dao, extract the values and return that array
2435    *
2436    * @param CRM_Core_DAO $dao
2437    *
2438    * @return array
2439    *   values for this query
2440    */
2441   public function store($dao) {
2442     $value = [];
2443
2444     foreach ($this->_element as $key => $dontCare) {
2445       if (property_exists($dao, $key)) {
2446         if (strpos($key, '-') !== FALSE) {
2447           $values = explode('-', $key);
2448           $lastElement = array_pop($values);
2449           $current = &$value;
2450           $cnt = count($values);
2451           $count = 1;
2452           foreach ($values as $v) {
2453             if (!array_key_exists($v, $current)) {
2454               $current[$v] = [];
2455             }
2456             //bad hack for im_provider
2457             if ($lastElement == 'provider_id') {
2458               if ($count < $cnt) {
2459                 $current = &$current[$v];
2460               }
2461               else {
2462                 $lastElement = "{$v}_{$lastElement}";
2463               }
2464             }
2465             else {
2466               $current = &$current[$v];
2467             }
2468             $count++;
2469           }
2470
2471           $current[$lastElement] = $dao->$key;
2472         }
2473         else {
2474           $value[$key] = $dao->$key;
2475         }
2476       }
2477     }
2478     return $value;
2479   }
2480
2481   /**
2482    * Getter for tables array.
2483    *
2484    * @return array
2485    */
2486   public function tables() {
2487     return $this->_tables;
2488   }
2489
2490   /**
2491    * Sometimes used to create the from clause, but, not reliably, set
2492    * this AND set tables.
2493    *
2494    * It's unclear the intent - there is a 'simpleFrom' clause which
2495    * takes whereTables into account & a fromClause which doesn't.
2496    *
2497    * logic may have eroded?
2498    *
2499    * @return array
2500    */
2501   public function whereTables() {
2502     return $this->_whereTables;
2503   }
2504
2505   /**
2506    * Generate the where clause (used in match contacts and permissions)
2507    *
2508    * @param array $params
2509    * @param array $fields
2510    * @param array $tables
2511    * @param $whereTables
2512    * @param bool $strict
2513    *
2514    * @return string
2515    * @throws \CRM_Core_Exception
2516    */
2517   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2518     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2519       FALSE, $strict
2520     );
2521
2522     $tables = array_merge($query->tables(), $tables);
2523     $whereTables = array_merge($query->whereTables(), $whereTables);
2524
2525     return $query->_whereClause;
2526   }
2527
2528   /**
2529    * Create the from clause.
2530    *
2531    * @param array $tables
2532    *   Tables that need to be included in this from clause. If null,
2533    *   return mimimal from clause (i.e. civicrm_contact).
2534    * @param array $inner
2535    *   Tables that should be inner-joined.
2536    * @param array $right
2537    *   Tables that should be right-joined.
2538    * @param bool $primaryLocation
2539    *   Search on primary location. See note below.
2540    * @param int $mode
2541    *   Determines search mode based on bitwise MODE_* constants.
2542    * @param string|NULL $apiEntity
2543    *   Determines search mode based on entity by string.
2544    *
2545    * The $primaryLocation flag only seems to be used when
2546    * locationType() has been called. This may be a search option
2547    * exposed, or perhaps it's a "search all details" approach which
2548    * predates decoupling of location types and primary fields?
2549    *
2550    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2551    *
2552    * @return string
2553    *   the from clause
2554    */
2555   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2556
2557     $from = ' FROM civicrm_contact contact_a';
2558     if (empty($tables)) {
2559       return $from;
2560     }
2561
2562     if (!empty($tables['civicrm_worldregion'])) {
2563       $tables = array_merge(['civicrm_country' => 1], $tables);
2564     }
2565
2566     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2567         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2568       $tables = array_merge(['civicrm_address' => 1],
2569         $tables
2570       );
2571     }
2572
2573     // add group_contact and group table is subscription history is present
2574     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2575       $tables = array_merge([
2576         'civicrm_group' => 1,
2577         'civicrm_group_contact' => 1,
2578       ],
2579         $tables
2580       );
2581     }
2582
2583     // to handle table dependencies of components
2584     CRM_Core_Component::tableNames($tables);
2585     // to handle table dependencies of hook injected tables
2586     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2587
2588     //format the table list according to the weight
2589     $info = CRM_Core_TableHierarchy::info();
2590
2591     foreach ($tables as $key => $value) {
2592       $k = 99;
2593       if (strpos($key, '-') !== FALSE) {
2594         $keyArray = explode('-', $key);
2595         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2596       }
2597       elseif (strpos($key, '_') !== FALSE) {
2598         $keyArray = explode('_', $key);
2599         if (is_numeric(array_pop($keyArray))) {
2600           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2601         }
2602         else {
2603           $k = CRM_Utils_Array::value($key, $info, 99);
2604         }
2605       }
2606       else {
2607         $k = CRM_Utils_Array::value($key, $info, 99);
2608       }
2609       $tempTable[$k . ".$key"] = $key;
2610     }
2611     ksort($tempTable);
2612     $newTables = [];
2613     foreach ($tempTable as $key) {
2614       $newTables[$key] = $tables[$key];
2615     }
2616
2617     $tables = $newTables;
2618
2619     foreach ($tables as $name => $value) {
2620       if (!$value) {
2621         continue;
2622       }
2623
2624       if (!empty($inner[$name])) {
2625         $side = 'INNER';
2626       }
2627       elseif (!empty($right[$name])) {
2628         $side = 'RIGHT';
2629       }
2630       else {
2631         $side = 'LEFT';
2632       }
2633
2634       if ($value != 1) {
2635         // if there is already a join statement in value, use value itself
2636         if (strpos($value, 'JOIN')) {
2637           $from .= " $value ";
2638         }
2639         else {
2640           $from .= " $side JOIN $name ON ( $value ) ";
2641         }
2642         continue;
2643       }
2644
2645       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2646     }
2647     return $from;
2648   }
2649
2650   /**
2651    * Get join statements for the from clause depending on entity type
2652    *
2653    * @param string $name
2654    * @param int $mode
2655    * @param string $side
2656    * @param string $primaryLocation
2657    * @return string
2658    */
2659   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2660     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2661     switch ($name) {
2662       case 'civicrm_address':
2663         //CRM-14263 further handling of address joins further down...
2664         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2665
2666       case 'civicrm_state_province':
2667         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2668         // but that will have been implied-in by the calling function.
2669         // test cover in testContactIDQuery
2670         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2671
2672       case 'civicrm_country':
2673         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2674         // but that will have been implied-in by the calling function.
2675         // test cover in testContactIDQuery
2676         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2677
2678       case 'civicrm_phone':
2679         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2680
2681       case 'civicrm_email':
2682         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2683
2684       case 'civicrm_im':
2685         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2686
2687       case 'im_provider':
2688         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2689         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2690         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2691         return $from;
2692
2693       case 'civicrm_openid':
2694         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2695
2696       case 'civicrm_worldregion':
2697         // We can be sure from the calling function that country will already be joined in.
2698         // we really don't need world_region - we could use a pseudoconstant for it.
2699         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2700
2701       case 'civicrm_location_type':
2702         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2703
2704       case 'civicrm_group':
2705         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2706
2707       case 'civicrm_group_contact':
2708         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2709
2710       case 'civicrm_group_contact_cache':
2711         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2712
2713       case 'civicrm_activity':
2714       case 'civicrm_activity_tag':
2715       case 'activity_type':
2716       case 'activity_status':
2717       case 'parent_id':
2718       case 'civicrm_activity_contact':
2719       case 'source_contact':
2720       case 'activity_priority':
2721         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2722
2723       case 'civicrm_entity_tag':
2724         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2725         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2726
2727       case 'civicrm_note':
2728         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2729         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2730
2731       case 'civicrm_subscription_history':
2732         $from = " $side JOIN civicrm_subscription_history";
2733         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2734         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2735
2736       case 'civicrm_relationship':
2737         if (self::$_relType == 'reciprocal') {
2738           if (self::$_relationshipTempTable) {
2739             // we have a temptable to join on
2740             $tbl = self::$_relationshipTempTable;
2741             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2742           }
2743           else {
2744             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2745             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2746             return $from;
2747           }
2748         }
2749         elseif (self::$_relType == 'b') {
2750           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2751           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2752         }
2753         else {
2754           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2755           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2756         }
2757
2758       case 'civicrm_log':
2759         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2760         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2761
2762       case 'civicrm_tag':
2763         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2764
2765       case 'civicrm_grant':
2766         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2767
2768       case 'civicrm_website':
2769         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2770
2771       case 'civicrm_campaign':
2772         //Move to default case if not in either mode.
2773         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2774           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2775         }
2776         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2777           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2778         }
2779         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2780           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2781         }
2782
2783       default:
2784         $locationTypeName = '';
2785         if (strpos($name, '-address') != 0) {
2786           $locationTypeName = 'address';
2787         }
2788         elseif (strpos($name, '-phone') != 0) {
2789           $locationTypeName = 'phone';
2790         }
2791         elseif (strpos($name, '-email') != 0) {
2792           $locationTypeName = 'email';
2793         }
2794         elseif (strpos($name, '-im') != 0) {
2795           $locationTypeName = 'im';
2796         }
2797         elseif (strpos($name, '-openid') != 0) {
2798           $locationTypeName = 'openid';
2799         }
2800
2801         if ($locationTypeName) {
2802           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2803           $parts = explode('-', $name);
2804           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2805           foreach ($locationTypes as $locationTypeID => $locationType) {
2806             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2807               $locationID = $locationTypeID;
2808             }
2809           }
2810           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2811         }
2812         else {
2813           $from = CRM_Core_Component::from($name, $mode, $side);
2814         }
2815         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2816
2817         return $from;
2818     }
2819   }
2820
2821   /**
2822    * WHERE / QILL clause for deleted_contacts
2823    *
2824    * @param array $values
2825    */
2826   public function deletedContacts($values) {
2827     list($_, $_, $value, $grouping, $_) = $values;
2828     if ($value) {
2829       // *prepend* to the relevant grouping as this is quite an important factor
2830       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2831     }
2832   }
2833
2834   /**
2835    * Where / qill clause for contact_type
2836    *
2837    * @param $values
2838    *
2839    * @throws \CRM_Core_Exception
2840    */
2841   public function contactType(&$values) {
2842     list($name, $op, $value, $grouping, $wildcard) = $values;
2843
2844     $subTypes = [];
2845     $clause = [];
2846
2847     // account for search builder mapping multiple values
2848     if (!is_array($value)) {
2849       $values = self::parseSearchBuilderString($value, 'String');
2850       if (is_array($values)) {
2851         $value = array_flip($values);
2852       }
2853     }
2854
2855     if (is_array($value)) {
2856       foreach ($value as $k => $v) {
2857         // fix for CRM-771
2858         if ($k) {
2859           $subType = NULL;
2860           $contactType = $k;
2861           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2862             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2863           }
2864
2865           if (!empty($subType)) {
2866             $subTypes[$subType] = 1;
2867           }
2868           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2869         }
2870       }
2871     }
2872     else {
2873       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2874       $contactType = $contactTypeANDSubType[0];
2875       $subType = $contactTypeANDSubType[1] ?? NULL;
2876       if (!empty($subType)) {
2877         $subTypes[$subType] = 1;
2878       }
2879       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2880     }
2881
2882     // fix for CRM-771
2883     if (!empty($clause)) {
2884       $quill = $clause;
2885       if ($op == 'IN' || $op == 'NOT IN') {
2886         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2887       }
2888       else {
2889         $type = array_pop($clause);
2890         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2891       }
2892
2893       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2894
2895       if (!empty($subTypes)) {
2896         $this->includeContactSubTypes($subTypes, $grouping);
2897       }
2898     }
2899   }
2900
2901   /**
2902    * Where / qill clause for contact_sub_type.
2903    *
2904    * @param array $values
2905    */
2906   public function contactSubType(&$values) {
2907     list($name, $op, $value, $grouping, $wildcard) = $values;
2908     $this->includeContactSubTypes($value, $grouping, $op);
2909   }
2910
2911   /**
2912    * @param $value
2913    * @param $grouping
2914    * @param string $op
2915    *
2916    * @throws \CRM_Core_Exception
2917    */
2918   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2919
2920     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2921       $op = key($value);
2922       $value = $value[$op];
2923     }
2924
2925     $clause = [];
2926     $alias = "contact_a.contact_sub_type";
2927     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2928
2929     $op = str_replace('IN', 'LIKE', $op);
2930     $op = str_replace('=', 'LIKE', $op);
2931     $op = str_replace('!', 'NOT ', $op);
2932
2933     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2934       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2935     }
2936     elseif (is_array($value)) {
2937       foreach ($value as $k => $v) {
2938         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2939       }
2940     }
2941     else {
2942       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2943     }
2944
2945     if (!empty($clause)) {
2946       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2947     }
2948     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2949   }
2950
2951   /**
2952    * Where / qill clause for groups.
2953    *
2954    * @param $values
2955    *
2956    * @throws \CRM_Core_Exception
2957    * @throws \Exception
2958    */
2959   public function group($values) {
2960     list($name, $op, $value, $grouping, $wildcard) = $values;
2961
2962     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2963     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2964       $op = key($value);
2965       $value = $value[$op];
2966     }
2967     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2968     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2969     // even one group might equate to multiple when looking at children so IN is simpler.
2970     // @todo - also look at != casting but there are rows below to review.
2971     $opReplacements = [
2972       'IS EMPTY' => 'IS NULL',
2973       'IS NOT EMPTY' => 'IS NOT NULL',
2974       '=' => 'IN',
2975     ];
2976     if (isset($opReplacements[$op])) {
2977       $op = $opReplacements[$op];
2978     }
2979
2980     if (strpos($op, 'NULL')) {
2981       $value = NULL;
2982     }
2983
2984     if (is_array($value) && count($value) > 1) {
2985       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2986         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2987       }
2988       $this->_useDistinct = TRUE;
2989     }
2990
2991     if (isset($value)) {
2992       $value = CRM_Utils_Array::value($op, $value, $value);
2993     }
2994
2995     if ($name === 'group_type') {
2996       $value = array_keys($this->getGroupsFromTypeCriteria($value));
2997     }
2998
2999     $regularGroupIDs = $smartGroupIDs = [];
3000     foreach ((array) $value as $id) {
3001       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3002         $smartGroupIDs[] = (int) $id;
3003       }
3004       else {
3005         $regularGroupIDs[] = (int) trim($id);
3006       }
3007     }
3008     $hasNonSmartGroups = count($regularGroupIDs);
3009
3010     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3011
3012     $statusJoinClause = $this->getGroupStatusClause($grouping);
3013     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3014     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3015     $groupClause = [];
3016     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3017       // include child groups IDs if any
3018       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3019       foreach ($childGroupIds as $key => $id) {
3020         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3021           $smartGroupIDs[] = $id;
3022           unset($childGroupIds[$key]);
3023         }
3024       }
3025       if (count($childGroupIds)) {
3026         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3027       }
3028
3029       if (empty($regularGroupIDs)) {
3030         if ($isGroupStatusSearch) {
3031           $regularGroupIDs = $smartGroupIDs;
3032         }
3033         // If it is still empty we want a filter that blocks all results.
3034         if (empty($regularGroupIDs)) {
3035           $regularGroupIDs = [0];
3036         }
3037       }
3038
3039       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3040       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3041
3042       // @todo consider just casting != to NOT IN & handling both together.
3043       if ($op === '!=') {
3044         $groupIds = '';
3045         if (!empty($regularGroupIDs)) {
3046           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3047         }
3048         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3049       }
3050       else {
3051         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3052       }
3053       $groupClause[] = "( {$clause} )";
3054
3055       if ($statusJoinClause) {
3056         $joinClause[] = "{$gcTable}.$statusJoinClause";
3057       }
3058       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3059     }
3060
3061     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3062     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3063     // as that info is in the group_contact table.
3064     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3065       $this->_groupUniqueKey = uniqid();
3066       $this->_groupKeys[] = $this->_groupUniqueKey;
3067       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3068       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3069       if (!empty($groupContactCacheClause)) {
3070         if ($isNotOp) {
3071           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3072           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3073           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3074           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3075           if (strpos($op, 'IN') !== FALSE) {
3076             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3077           }
3078           else {
3079             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3080           }
3081         }
3082         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3083       }
3084     }
3085
3086     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3087     if (!empty($groupClause)) {
3088       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3089     }
3090
3091     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3092     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3093     if (strpos($op, 'NULL') === FALSE) {
3094       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3095     }
3096   }
3097
3098   /**
3099    * @return array
3100    */
3101   public function getGroupCacheTableKeys() {
3102     return $this->_groupKeys;
3103   }
3104
3105   /**
3106    * Function translates selection of group type into a list of groups.
3107    * @param $value
3108    *
3109    * @return array
3110    */
3111   public function getGroupsFromTypeCriteria($value) {
3112     $groupIds = [];
3113     foreach ((array) $value as $groupTypeValue) {
3114       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3115       $groupIds = ($groupIds + $groupList);
3116     }
3117     return $groupIds;
3118   }
3119
3120   /**
3121    * Prime smart group cache for smart groups in the search, and join
3122    * civicrm_group_contact_cache table into the query.
3123    *
3124    * @param array $groups IDs of groups specified in search criteria.
3125    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3126    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3127    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3128    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3129    *
3130    * @return string WHERE clause component for smart group criteria.
3131    * @throws \CRM_Core_Exception
3132    */
3133   public function addGroupContactCache($groups, $tableAlias, $joinTable, $op, $joinColumn = 'id') {
3134     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3135     $groupsIds = $groups;
3136
3137     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3138     if (!empty($operator[$op]) && is_array($groups)) {
3139       $op = $operator[$op];
3140     }
3141     if (!$isNullOp && !$groups) {
3142       return NULL;
3143     }
3144     elseif (strpos($op, 'IN') !== FALSE) {
3145       $groups = [$op => $groups];
3146     }
3147     elseif (is_array($groups) && count($groups)) {
3148       $groups = ['IN' => $groups];
3149     }
3150
3151     // Find all the groups that are part of a saved search.
3152     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3153     $sql = "
3154 SELECT id, cache_date, saved_search_id, children
3155 FROM   civicrm_group
3156 WHERE  $smartGroupClause
3157   AND  ( saved_search_id != 0
3158    OR    saved_search_id IS NOT NULL
3159    OR    children IS NOT NULL )
3160 ";
3161
3162     $group = CRM_Core_DAO::executeQuery($sql);
3163
3164     while ($group->fetch()) {
3165       $this->_useDistinct = TRUE;
3166       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3167         CRM_Contact_BAO_GroupContactCache::load($group);
3168       }
3169     }
3170     if ($group->N == 0 && $op != 'NOT IN') {
3171       return NULL;
3172     }
3173
3174     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3175
3176     if ($op == 'NOT IN') {
3177       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3178     }
3179     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3180   }
3181
3182   /**
3183    * Where / qill clause for cms users
3184    *
3185    * @param $values
3186    */
3187   public function ufUser(&$values) {
3188     list($name, $op, $value, $grouping, $wildcard) = $values;
3189
3190     if ($value == 1) {
3191       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3192
3193       $this->_qill[$grouping][] = ts('CMS User');
3194     }
3195     elseif ($value == 0) {
3196       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3197
3198       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3199       $this->_qill[$grouping][] = ts('Not a CMS User');
3200     }
3201   }
3202
3203   /**
3204    * All tag search specific.
3205    *
3206    * @param array $values
3207    *
3208    * @throws \CRM_Core_Exception
3209    */
3210   public function tagSearch(&$values) {
3211     list($name, $op, $value, $grouping, $wildcard) = $values;
3212
3213     $op = "LIKE";
3214     $value = "%{$value}%";
3215     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3216
3217     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3218     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3219
3220     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3221     $tTable = "`civicrm_tag-" . uniqid() . "`";
3222
3223     if ($useAllTagTypes[2]) {
3224       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3225         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3226             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3227
3228       // search tag in cases
3229       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3230       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3231       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3232         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3233             LEFT JOIN civicrm_case
3234             ON (civicrm_case_contact.case_id = civicrm_case.id
3235                 AND civicrm_case.is_deleted = 0 )
3236             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3237             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3238       // search tag in activities
3239       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3240       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3241       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3242       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3243
3244       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3245         = " LEFT JOIN civicrm_activity_contact
3246             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3247             LEFT JOIN civicrm_activity
3248             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3249             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3250             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3251             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3252
3253       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3254       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3255     }
3256     else {
3257       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3258       $tTable = "`civicrm_tag-" . uniqid() . "`";
3259       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3260       {$etTable}.entity_table = 'civicrm_contact' )
3261                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3262
3263       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3264       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3265     }
3266   }
3267
3268   /**
3269    * Where / qill clause for tag.
3270    *
3271    * @param array $values
3272    *
3273    * @throws \CRM_Core_Exception
3274    */
3275   public function tag(&$values) {
3276     list($name, $op, $value, $grouping, $wildcard) = $values;
3277
3278     // API/Search Builder format array(operator => array(values))
3279     if (is_array($value)) {
3280       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3281         $op = key($value);
3282         $value = $value[$op];
3283       }
3284       if (count($value) > 1) {
3285         $this->_useDistinct = TRUE;
3286       }
3287     }
3288
3289     if (strpos($op, 'NULL') || strpos($op, 'EMPTY')) {
3290       $value = NULL;
3291     }
3292
3293     $tagTree = CRM_Core_BAO_Tag::getChildTags();
3294     foreach ((array) $value as $tagID) {
3295       if (!empty($tagTree[$tagID])) {
3296         // make sure value is an array here (see CORE-2502)
3297         $value = array_unique(array_merge((array) $value, $tagTree[$tagID]));
3298       }
3299     }
3300
3301     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3302
3303     // implode array, then remove all spaces
3304     $value = str_replace(' ', '', implode(',', (array) $value));
3305     if (!empty($value)) {
3306       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3307     }
3308
3309     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3310     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3311
3312     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3313
3314     if (!empty($useAllTagTypes[2])) {
3315       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3316         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3317
3318       // search tag in cases
3319       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3320       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3321       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3322
3323       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3324         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3325             LEFT JOIN civicrm_case
3326             ON (civicrm_case_contact.case_id = civicrm_case.id
3327                 AND civicrm_case.is_deleted = 0 )
3328             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3329       // search tag in activities
3330       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3331       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3332         = " LEFT JOIN civicrm_activity_contact
3333             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3334             LEFT JOIN civicrm_activity
3335             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3336             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3337             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3338
3339       // CRM-10338
3340       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3341         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3342       }
3343       else {
3344         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3345       }
3346     }
3347     else {
3348       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3349         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3350
3351       // CRM-10338
3352       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3353         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3354         $op = str_replace('EMPTY', 'NULL', $op);
3355         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3356       }
3357       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3358       elseif ($op == '!=') {
3359         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3360       }
3361       elseif ($op == '=' || strstr($op, 'IN')) {
3362         $op = ($op == '=') ? 'IN' : $op;
3363         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3364       }
3365     }
3366     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3367   }
3368
3369   /**
3370    * Where/qill clause for notes
3371    *
3372    * @param array $values
3373    *
3374    * @throws \CRM_Core_Exception
3375    */
3376   public function notes(&$values) {
3377     list($name, $op, $value, $grouping, $wildcard) = $values;
3378
3379     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3380     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3381     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3382
3383     $this->_useDistinct = TRUE;
3384
3385     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3386       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3387
3388     $n = trim($value);
3389     $value = CRM_Core_DAO::escapeString($n);
3390     if ($wildcard) {
3391       if (strpos($value, '%') === FALSE) {
3392         $value = "%$value%";
3393       }
3394       $op = 'LIKE';
3395     }
3396     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3397       $value = NULL;
3398     }
3399
3400     $label = NULL;
3401     $clauses = [];
3402     if ($noteOption % 2 == 0) {
3403       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3404       $label = ts('Note: Body Only');
3405     }
3406     if ($noteOption % 3 == 0) {
3407       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3408       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3409     }
3410     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3411     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3412     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3413   }
3414
3415   /**
3416    * @param string $name
3417    * @param $op
3418    * @param $grouping
3419    *
3420    * @return bool
3421    */
3422   public function nameNullOrEmptyOp($name, $op, $grouping) {
3423     switch ($op) {
3424       case 'IS NULL':
3425       case 'IS NOT NULL':
3426         $this->_where[$grouping][] = "contact_a.$name $op";
3427         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3428         return TRUE;
3429
3430       case 'IS EMPTY':
3431         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3432         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3433         return TRUE;
3434
3435       case 'IS NOT EMPTY':
3436         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3437         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3438         return TRUE;
3439
3440       default:
3441         return FALSE;
3442     }
3443   }
3444
3445   /**
3446    * Where / qill clause for sort_name
3447    *
3448    * @param array $values
3449    */
3450   public function sortName(&$values) {
3451     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3452
3453     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3454     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3455       return;
3456     }
3457
3458     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3459
3460     if (!strlen($value)) {
3461       return;
3462     }
3463
3464     $config = CRM_Core_Config::singleton();
3465
3466     $sub = [];
3467
3468     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3469     $subGlue = ' OR ';
3470
3471     $firstChar = substr($value, 0, 1);
3472     $lastChar = substr($value, -1, 1);
3473     $quotes = ["'", '"'];
3474     // If string is quoted, strip quotes and otherwise don't alter it
3475     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3476       $value = trim($value, implode('', $quotes));
3477     }
3478     // Replace spaces with wildcards for a LIKE operation
3479     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3480     // Also need to check if there is space in between sort name.
3481     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') === TRUE) {
3482       $value = str_replace(' ', '%', $value);
3483     }
3484     $value = CRM_Core_DAO::escapeString(trim($value));
3485     if (strlen($value)) {
3486       $fieldsub = [];
3487       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3488       if ($fieldName == 'sort_name') {
3489         $wc = "contact_a.sort_name";
3490       }
3491       else {
3492         $wc = "contact_a.display_name";
3493       }
3494       $fieldsub[] = " ( $wc $op $value )";
3495       if ($config->includeNickNameInName) {
3496         $wc = "contact_a.nick_name";
3497         $fieldsub[] = " ( $wc $op $value )";
3498       }
3499       if ($config->includeEmailInName) {
3500         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3501       }
3502       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3503     }
3504
3505     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3506
3507     $this->_where[$grouping][] = $sub;
3508     if ($config->includeEmailInName) {
3509       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3510       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3511     }
3512     else {
3513       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3514     }
3515   }
3516
3517   /**
3518    * Where/qill clause for greeting fields.
3519    *
3520    * @param array $values
3521    *
3522    * @throws \CRM_Core_Exception
3523    */
3524   public function greetings(&$values) {
3525     list($name, $op, $value, $grouping, $wildcard) = $values;
3526     $name .= '_display';
3527
3528     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3529     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3530     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3531   }
3532
3533   /**
3534    * Where / qill clause for email
3535    *
3536    * @param array $values
3537    * @param string $isForcePrimaryOnly
3538    *
3539    * @throws \CRM_Core_Exception
3540    */
3541   protected function email(&$values, $isForcePrimaryOnly) {
3542     list($name, $op, $value, $grouping, $wildcard) = $values;
3543     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3544
3545     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3546     // so following code is use build whereClause for contact's primart email id
3547     if (!empty($isForcePrimaryOnly)) {
3548       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3549     }
3550     // @todo - this should come from the $this->_fields array
3551     $dbName = $name === 'email_id' ? 'id' : $name;
3552
3553     if (is_array($value) || $name === 'email_id') {
3554       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3555       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3556       return;
3557     }
3558
3559     // Is this ever hit now? Ideally ensure always an array & handle above.
3560     $n = trim($value);
3561     if ($n) {
3562       if (substr($n, 0, 1) == '"' &&
3563         substr($n, -1, 1) == '"'
3564       ) {
3565         $n = substr($n, 1, -1);
3566         $value = CRM_Core_DAO::escapeString($n);
3567         $op = '=';
3568       }
3569       else {
3570         $value = self::getWildCardedValue($wildcard, $op, $n);
3571       }
3572       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3573       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3574     }
3575     else {
3576       $this->_qill[$grouping][] = ts('Email') . " $op ";
3577       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3578     }
3579   }
3580
3581   /**
3582    * Where / qill clause for phone number
3583    *
3584    * @param array $values
3585    *
3586    * @throws \CRM_Core_Exception
3587    */
3588   public function phone_numeric(&$values) {
3589     list($name, $op, $value, $grouping, $wildcard) = $values;
3590     // Strip non-numeric characters; allow wildcards
3591     $number = preg_replace('/[^\d%]/', '', $value);
3592     if ($number) {
3593       if (strpos($number, '%') === FALSE) {
3594         $number = "%$number%";
3595       }
3596
3597       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3598       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3599       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3600     }
3601   }
3602
3603   /**
3604    * Where / qill clause for phone type/location
3605    *
3606    * @param array $values
3607    *
3608    * @throws \CRM_Core_Exception
3609    */
3610   public function phone_option_group($values) {
3611     list($name, $op, $value, $grouping, $wildcard) = $values;
3612     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3613     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3614     $optionName = $options[$value];
3615     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3616     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3617     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3618   }
3619
3620   /**
3621    * Where / qill clause for street_address.
3622    *
3623    * @param array $values
3624    *
3625    * @throws \CRM_Core_Exception
3626    */
3627   public function street_address(&$values) {
3628     list($name, $op, $value, $grouping) = $values;
3629
3630     if (!$op) {
3631       $op = 'LIKE';
3632     }
3633
3634     $n = trim($value);
3635
3636     if ($n) {
3637       if (strpos($value, '%') === FALSE) {
3638         // only add wild card if not there
3639         $value = "%{$value}%";
3640       }
3641       $op = 'LIKE';
3642       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3643       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3644     }
3645     else {
3646       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3647       $this->_qill[$grouping][] = ts('Street') . " $op ";
3648     }
3649
3650     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3651   }
3652
3653   /**
3654    * Where / qill clause for street_unit.
3655    *
3656    * @param array $values
3657    *
3658    * @throws \CRM_Core_Exception
3659    */
3660   public function street_number(&$values) {
3661     list($name, $op, $value, $grouping, $wildcard) = $values;
3662
3663     if (!$op) {
3664       $op = '=';
3665     }
3666
3667     $n = trim($value);
3668
3669     if (strtolower($n) == 'odd') {
3670       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3671       $this->_qill[$grouping][] = ts('Street Number is odd');
3672     }
3673     elseif (strtolower($n) == 'even') {
3674       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3675       $this->_qill[$grouping][] = ts('Street Number is even');
3676     }
3677     else {
3678       $value = $n;
3679       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3680       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3681     }
3682
3683     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3684   }
3685
3686   /**
3687    * Where / qill clause for sorting by character.
3688    *
3689    * @param array $values
3690    */
3691   public function sortByCharacter(&$values) {
3692     list($name, $op, $value, $grouping, $wildcard) = $values;
3693
3694     $name = trim($value);
3695     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3696     $this->_where[$grouping][] = $cond;
3697     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3698   }
3699
3700   /**
3701    * Where / qill clause for including contact ids.
3702    */
3703   public function includeContactIDs() {
3704     if (!$this->_includeContactIds || empty($this->_params)) {
3705       return;
3706     }
3707
3708     $contactIds = [];
3709     foreach ($this->_params as $id => $values) {
3710       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3711         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3712       }
3713     }
3714     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3715     if (!empty($contactIds)) {
3716       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3717     }
3718   }
3719
3720   /**
3721    * Where / qill clause for postal code.
3722    *
3723    * @param array $values
3724    *
3725    * @throws \CRM_Core_Exception
3726    */
3727   public function postalCode(&$values) {
3728     // skip if the fields dont have anything to do with postal_code
3729     if (empty($this->_fields['postal_code'])) {
3730       return;
3731     }
3732
3733     list($name, $op, $value, $grouping, $wildcard) = $values;
3734
3735     // Handle numeric postal code range searches properly by casting the column as numeric
3736     if (is_numeric($value)) {
3737       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3738       $val = CRM_Utils_Type::escape($value, 'Integer');
3739     }
3740     else {
3741       $field = 'civicrm_address.postal_code';
3742       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3743       if (!is_array($value)) {
3744         $val = CRM_Utils_Type::escape($value, 'String');
3745       }
3746       else {
3747         // Do we need to escape values here? I would expect buildClause does.
3748         $val = $value;
3749       }
3750     }
3751
3752     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3753
3754     if ($name == 'postal_code') {
3755       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3756       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3757     }
3758     elseif ($name == 'postal_code_low') {
3759       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3760       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3761     }
3762     elseif ($name == 'postal_code_high') {
3763       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3764       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3765     }
3766   }
3767
3768   /**
3769    * Where / qill clause for location type.
3770    *
3771    * @param array $values
3772    * @param null $status
3773    *
3774    * @return string
3775    */
3776   public function locationType(&$values, $status = NULL) {
3777     list($name, $op, $value, $grouping, $wildcard) = $values;
3778
3779     if (is_array($value)) {
3780       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3781       $this->_tables['civicrm_address'] = 1;
3782       $this->_whereTables['civicrm_address'] = 1;
3783
3784       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3785       $names = [];
3786       foreach ($value as $id) {
3787         $names[] = $locationType[$id];
3788       }
3789
3790       $this->_primaryLocation = FALSE;
3791
3792       if (!$status) {
3793         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3794       }
3795       else {
3796         return implode(' ' . ts('or') . ' ', $names);
3797       }
3798     }
3799   }
3800
3801   /**
3802    * @param $values
3803    * @param bool $fromStateProvince
3804    *
3805    * @return array|NULL
3806    * @throws \CRM_Core_Exception
3807    */
3808   public function country(&$values, $fromStateProvince = TRUE) {
3809     list($name, $op, $value, $grouping, $wildcard) = $values;
3810
3811     if (!$fromStateProvince) {
3812       $stateValues = $this->getWhereValues('state_province', $grouping);
3813       if (!empty($stateValues)) {
3814         // return back to caller if there are state province values
3815         // since that handles this case
3816         return NULL;
3817       }
3818     }
3819
3820     $countryClause = $countryQill = NULL;
3821     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3822       $this->_tables['civicrm_address'] = 1;
3823       $this->_whereTables['civicrm_address'] = 1;
3824
3825       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3826       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3827       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3828
3829       if (!$fromStateProvince) {
3830         $this->_where[$grouping][] = $countryClause;
3831         $this->_qill[$grouping][] = $countryQill;
3832       }
3833     }
3834
3835     if ($fromStateProvince) {
3836       if (!empty($countryClause)) {
3837         return [
3838           $countryClause,
3839           " ...AND... " . $countryQill,
3840         ];
3841       }
3842       else {
3843         return [NULL, NULL];
3844       }
3845     }
3846   }
3847
3848   /**
3849    * Where / qill clause for county (if present).
3850    *
3851    * @param array $values
3852    * @param null $status
3853    *
3854    * @return string
3855    */
3856   public function county(&$values, $status = NULL) {
3857     list($name, $op, $value, $grouping, $wildcard) = $values;
3858
3859     if (!is_array($value)) {
3860       // force the county to be an array
3861       $value = [$value];
3862     }
3863
3864     // check if the values are ids OR names of the counties
3865     $inputFormat = 'id';
3866     foreach ($value as $v) {
3867       if (!is_numeric($v)) {
3868         $inputFormat = 'name';
3869         break;
3870       }
3871     }
3872     $names = [];
3873     if ($op == '=') {
3874       $op = 'IN';
3875     }
3876     elseif ($op == '!=') {
3877       $op = 'NOT IN';
3878     }
3879     else {
3880       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3881       $op = str_replace('EMPTY', 'NULL', $op);
3882     }
3883
3884     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3885       $clause = "civicrm_address.county_id $op";
3886     }
3887     elseif ($inputFormat == 'id') {
3888       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3889
3890       $county = CRM_Core_PseudoConstant::county();
3891       foreach ($value as $id) {
3892         $names[] = $county[$id] ?? NULL;
3893       }
3894     }
3895     else {
3896       $inputClause = [];
3897       $county = CRM_Core_PseudoConstant::county();
3898       foreach ($value as $name) {
3899         $name = trim($name);
3900         $inputClause[] = CRM_Utils_Array::key($name, $county);
3901       }
3902       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3903       $names = $value;
3904     }
3905     $this->_tables['civicrm_address'] = 1;
3906     $this->_whereTables['civicrm_address'] = 1;
3907
3908     $this->_where[$grouping][] = $clause;
3909     if (!$status) {
3910       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3911     }
3912     else {
3913       return implode(' ' . ts('or') . ' ', $names);
3914     }
3915   }
3916
3917   /**
3918    * Where / qill clause for state/province AND country (if present).
3919    *
3920    * @param array $values
3921    * @param null $status
3922    *
3923    * @return string
3924    * @throws \CRM_Core_Exception
3925    */
3926   public function stateProvince(&$values, $status = NULL) {
3927     list($name, $op, $value, $grouping, $wildcard) = $values;
3928
3929     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3930     $this->_tables['civicrm_address'] = 1;
3931     $this->_whereTables['civicrm_address'] = 1;
3932
3933     $countryValues = $this->getWhereValues('country', $grouping);
3934     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3935     if ($countryClause) {
3936       $clause = "( $stateClause AND $countryClause )";
3937     }
3938     else {
3939       $clause = $stateClause;
3940     }
3941
3942     $this->_where[$grouping][] = $clause;
3943     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3944     if (!$status) {
3945       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3946     }
3947     else {
3948       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3949     }
3950   }
3951
3952   /**
3953    * Where / qill clause for change log.
3954    *
3955    * @param array $values
3956    */
3957   public function changeLog(&$values) {
3958     list($name, $op, $value, $grouping, $wildcard) = $values;
3959
3960     $targetName = $this->getWhereValues('changed_by', $grouping);
3961     if (!$targetName) {
3962       return;
3963     }
3964
3965     $name = trim($targetName[2]);
3966     $name = CRM_Core_DAO::escapeString($name);
3967     $name = $targetName[4] ? "%$name%" : $name;
3968     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3969     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3970     $fieldTitle = ts('Altered By');
3971
3972     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3973     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3974       1 => $fieldTitle,
3975       2 => $qillop,
3976       3 => $qillVal,
3977     ]);
3978   }
3979
3980   /**
3981    * @param $values
3982    *
3983    * @throws \CRM_Core_Exception
3984    */
3985   public function modifiedDates($values) {
3986     $this->_useDistinct = TRUE;
3987
3988     // CRM-11281, default to added date if not set
3989     $fieldTitle = ts('Added Date');
3990     $fieldName = 'created_date';
3991     foreach (array_keys($this->_params) as $id) {
3992       if ($this->_params[$id][0] == 'log_date') {
3993         if ($this->_params[$id][2] == 2) {
3994           $fieldTitle = ts('Modified Date');
3995           $fieldName = 'modified_date';
3996         }
3997       }
3998     }
3999
4000     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
4001
4002     self::$_openedPanes[ts('Change Log')] = TRUE;
4003   }
4004
4005   /**
4006    * @param $values
4007    *
4008    * @throws \CRM_Core_Exception
4009    */
4010   public function demographics(&$values) {
4011     list($name, $op, $value, $grouping, $wildcard) = $values;
4012
4013     if (($name == 'age_low') || ($name == 'age_high')) {
4014       $this->ageRangeQueryBuilder($values,
4015         'contact_a', 'age', 'birth_date', ts('Age')
4016       );
4017     }
4018     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4019
4020       $this->dateQueryBuilder($values,
4021         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4022       );
4023     }
4024     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4025
4026       $this->dateQueryBuilder($values,
4027         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4028       );
4029     }
4030
4031     self::$_openedPanes[ts('Demographics')] = TRUE;
4032   }
4033
4034   /**
4035    * @param $values
4036    */
4037   public function privacy(&$values) {
4038     list($name, $op, $value, $grouping) = $values;
4039     if (is_array($value)) {
4040       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4041         $op = key($value);
4042         $value = $value[$op];
4043       }
4044     }
4045     $field = $this->_fields[$name] ?? NULL;
4046     CRM_Utils_Type::validate($value, 'Integer');
4047     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4048     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4049     $title = $field ? $field['title'] : $name;
4050     $this->_qill[$grouping][] = "$title $op $value";
4051   }
4052
4053   /**
4054    * @param $values
4055    */
4056   public function privacyOptions($values) {
4057     list($name, $op, $value, $grouping, $wildcard) = $values;
4058
4059     if (empty($value) || !is_array($value)) {
4060       return;
4061     }
4062
4063     // get the operator and toggle values
4064     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4065     $operator = 'OR';
4066     if ($opValues &&
4067       strtolower($opValues[2] == 'AND')
4068     ) {
4069       // @todo this line is logially unreachable
4070       $operator = 'AND';
4071     }
4072
4073     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4074     $compareOP = '!';
4075     if ($toggleValues &&
4076       $toggleValues[2] == 2
4077     ) {
4078       $compareOP = '';
4079     }
4080
4081     $clauses = [];
4082     $qill = [];
4083     foreach ($value as $dontCare => $pOption) {
4084       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4085       $field = $this->_fields[$pOption] ?? NULL;
4086       $title = $field ? $field['title'] : $pOption;
4087       $qill[] = " $title = 1 ";
4088     }
4089
4090     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4091     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4092   }
4093
4094   /**
4095    * @param $values
4096    *
4097    * @throws \CRM_Core_Exception
4098    */
4099   public function preferredCommunication(&$values) {
4100     list($name, $op, $value, $grouping, $wildcard) = $values;
4101
4102     if (!is_array($value)) {
4103       $value = str_replace(['(', ')'], '', explode(",", $value));
4104     }
4105     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4106       $op = key($value);
4107       $value = $value[$op];
4108     }
4109     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4110
4111     if (self::caseImportant($op)) {
4112       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4113       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4114       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4115     }
4116
4117     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4118     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4119   }
4120
4121   /**
4122    * Where / qill clause for relationship.
4123    *
4124    * @param array $values
4125    */
4126   public function relationship(&$values) {
4127     list($name, $op, $value, $grouping, $wildcard) = $values;
4128     if ($this->_relationshipValuesAdded) {
4129       return;
4130     }
4131     // also get values array for relation_target_name
4132     // for relationship search we always do wildcard
4133     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4134     $description = $this->getWhereValues('relation_description', $grouping);
4135     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4136     $relStatus = $this->getWhereValues('relation_status', $grouping);
4137     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4138
4139     $nameClause = $name = NULL;
4140     if ($targetName) {
4141       $name = trim($targetName[2]);
4142       if (substr($name, 0, 1) == '"' &&
4143         substr($name, -1, 1) == '"'
4144       ) {
4145         $name = substr($name, 1, -1);
4146         $name = CRM_Core_DAO::escapeString($name);
4147         $nameClause = "= '$name'";
4148       }
4149       else {
4150         $name = CRM_Core_DAO::escapeString($name);
4151         $nameClause = "LIKE '%{$name}%'";
4152       }
4153     }
4154
4155     $relTypes = $relTypesIds = [];
4156     if (!empty($relationType)) {
4157       $relationType[2] = (array) $relationType[2];
4158       foreach ($relationType[2] as $relType) {
4159         $rel = explode('_', $relType);
4160         self::$_relType = $rel[1];
4161         $params = ['id' => $rel[0]];
4162         $typeValues = [];
4163         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4164         if (!empty($rTypeValue)) {
4165           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4166             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4167             self::$_relType = 'reciprocal';
4168           }
4169           $relTypesIds[] = $rel[0];
4170           $relTypes[] = $relType;
4171         }
4172       }
4173     }
4174
4175     // if we are creating a temp table we build our own where for the relationship table
4176     $relationshipTempTable = NULL;
4177     if (self::$_relType == 'reciprocal') {
4178       $where = [];
4179       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4180         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4181         ->getName();
4182       if ($nameClause) {
4183         $where[$grouping][] = " sort_name $nameClause ";
4184       }
4185       $groupJoinTable = "civicrm_relationship";
4186       $groupJoinColumn = "contact_id_alt";
4187     }
4188     else {
4189       $where = &$this->_where;
4190       if ($nameClause) {
4191         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4192       }
4193       $groupJoinTable = "contact_b";
4194       $groupJoinColumn = "id";
4195     }
4196     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4197     if ($nameClause || !$targetGroup) {
4198       if (!empty($relationType)) {
4199         $relQill = '';
4200         foreach ($relTypes as $rel) {
4201           if (!empty($relQill)) {
4202             $relQill .= ' OR ';
4203           }
4204           $relQill .= $allRelationshipType[$rel];
4205         }
4206         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4207       }
4208       elseif ($name) {
4209         $this->_qill[$grouping][] = $name;
4210       }
4211     }
4212
4213     //check to see if the target contact is in specified group
4214     if ($targetGroup) {
4215       //add contacts from static groups
4216       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4217         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4218       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4219         implode(",", $targetGroup[2]) . ") ) ";
4220
4221       //add contacts from saved searches
4222       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4223
4224       //set the group where clause
4225       if ($ssWhere) {
4226         $groupWhere[] = "( " . $ssWhere . " )";
4227       }
4228       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4229
4230       //Get the names of the target groups for the qill
4231       $groupNames = CRM_Core_PseudoConstant::group();
4232       $qillNames = [];
4233       foreach ($targetGroup[2] as $groupId) {
4234         if (array_key_exists($groupId, $groupNames)) {
4235           $qillNames[] = $groupNames[$groupId];
4236         }
4237       }
4238       if (!empty($relationType)) {
4239         $relQill = '';
4240         foreach ($relTypes as $rel) {
4241           if (!empty($relQill)) {
4242             $relQill .= ' OR ';
4243           }
4244           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4245         }
4246         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4247       }
4248       else {
4249         $this->_qill[$grouping][] = implode(", ", $qillNames);
4250       }
4251     }
4252
4253     // Description
4254     if (!empty($description[2]) && trim($description[2])) {
4255       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4256       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4257       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4258     }
4259
4260     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4261     $today = date('Ymd');
4262     //check for active, inactive and all relation status
4263     if (empty($relStatus[2])) {
4264       $where[$grouping][] = "(
4265 civicrm_relationship.is_active = 1 AND
4266 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4267 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4268 )";
4269       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4270     }
4271     elseif ($relStatus[2] == 1) {
4272       $where[$grouping][] = "(
4273 civicrm_relationship.is_active = 0 OR
4274 civicrm_relationship.end_date < {$today} OR
4275 civicrm_relationship.start_date > {$today}
4276 )";
4277       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4278     }
4279
4280     $onlyDeleted = 0;
4281     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4282       $onlyDeleted = 1;
4283     }
4284     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4285
4286     $this->addRelationshipPermissionClauses($grouping, $where);
4287     $this->addRelationshipDateClauses($grouping, $where);
4288     $this->addRelationshipActivePeriodClauses($grouping, $where);
4289     if (!empty($relTypes)) {
4290       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4291     }
4292     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4293     $this->_useDistinct = TRUE;
4294     $this->_relationshipValuesAdded = TRUE;
4295     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4296     // join on that,
4297     if ($relationshipTempTable) {
4298       $whereClause = '';
4299       if (!empty($where[$grouping])) {
4300         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4301         $whereClause = str_replace('contact_b', 'c', $whereClause);
4302       }
4303       $sql = "
4304         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4305           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4306             FROM civicrm_relationship
4307             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4308             $whereClause )
4309           UNION
4310             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4311             FROM civicrm_relationship
4312             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4313             $whereClause )
4314       ";
4315       CRM_Core_DAO::executeQuery($sql);
4316     }
4317   }
4318
4319   /**
4320    * Add relationship permission criteria to where clause.
4321    *
4322    * @param string $grouping
4323    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4324    *   Not the main query.
4325    */
4326   public function addRelationshipPermissionClauses($grouping, &$where) {
4327     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4328     if ($relPermission) {
4329       if (!is_array($relPermission[2])) {
4330         // this form value was scalar in previous versions of Civi
4331         $relPermission[2] = [$relPermission[2]];
4332       }
4333       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4334
4335       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4336
4337       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4338       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4339     }
4340   }
4341
4342   /**
4343    * Add start & end date criteria in
4344    * @param string $grouping
4345    * @param array $where
4346    *   = array to add where clauses to, in case you are generating a temp table.
4347    * not the main query.
4348    */
4349   public function addRelationshipDateClauses($grouping, &$where) {
4350     foreach (['start_date', 'end_date'] as $dateField) {
4351       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4352       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4353       if (!empty($dateValueLow)) {
4354         $date = date('Ymd', strtotime($dateValueLow[2]));
4355         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4356         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4357       }
4358       if (!empty($dateValueHigh)) {
4359         $date = date('Ymd', strtotime($dateValueHigh[2]));
4360         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4361         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4362       }
4363     }
4364   }
4365
4366   /**
4367    * Add start & end active period criteria in
4368    * @param string $grouping
4369    * @param array $where
4370    *   = array to add where clauses to, in case you are generating a temp table.
4371    * not the main query.
4372    */
4373   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4374     $dateValues = [];
4375     $dateField = 'active_period_date';
4376
4377     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4378     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4379     $dateValueLowFormated = $dateValueHighFormated = NULL;
4380     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4381       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4382       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4383       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4384     }
4385     elseif (!empty($dateValueLow)) {
4386       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4387       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4388     }
4389     elseif (!empty($dateValueHigh)) {
4390       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4391       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4392     }
4393
4394     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4395       $where[$grouping][] = $activePeriodClauses;
4396     }
4397   }
4398
4399   /**
4400    * Get start & end active period criteria
4401    *
4402    * @param $from
4403    * @param $to
4404    * @param $forceTableName
4405    *
4406    * @return string
4407    */
4408   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4409     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4410     if (!is_null($from) && !is_null($to)) {
4411       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4412                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4413                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4414                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4415                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4416                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4417     }
4418     elseif (!is_null($from)) {
4419       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4420                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4421                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4422     }
4423     elseif (!is_null($to)) {
4424       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4425                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4426                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4427     }
4428   }
4429
4430   /**
4431    * Default set of return properties.
4432    *
4433    * @param int $mode
4434    *
4435    * @return array
4436    *   derault return properties
4437    */
4438   public static function &defaultReturnProperties($mode = 1) {
4439     if (!isset(self::$_defaultReturnProperties)) {
4440       self::$_defaultReturnProperties = [];
4441     }
4442
4443     if (!isset(self::$_defaultReturnProperties[$mode])) {
4444       // add activity return properties
4445       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4446         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4447       }
4448       else {
4449         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4450       }
4451
4452       if (empty(self::$_defaultReturnProperties[$mode])) {
4453         self::$_defaultReturnProperties[$mode] = [
4454           'image_URL' => 1,
4455           'legal_identifier' => 1,
4456           'external_identifier' => 1,
4457           'contact_type' => 1,
4458           'contact_sub_type' => 1,
4459           'sort_name' => 1,
4460           'display_name' => 1,
4461           'preferred_mail_format' => 1,
4462           'nick_name' => 1,
4463           'first_name' => 1,
4464           'middle_name' => 1,
4465           'last_name' => 1,
4466           'prefix_id' => 1,
4467           'suffix_id' => 1,
4468           'formal_title' => 1,
4469           'communication_style_id' => 1,
4470           'birth_date' => 1,
4471           'gender_id' => 1,
4472           'street_address' => 1,
4473           'supplemental_address_1' => 1,
4474           'supplemental_address_2' => 1,
4475           'supplemental_address_3' => 1,
4476           'city' => 1,
4477           'postal_code' => 1,
4478           'postal_code_suffix' => 1,
4479           'state_province' => 1,
4480           'country' => 1,
4481           'world_region' => 1,
4482           'geo_code_1' => 1,
4483           'geo_code_2' => 1,
4484           'email' => 1,
4485           'on_hold' => 1,
4486           'phone' => 1,
4487           'im' => 1,
4488           'household_name' => 1,
4489           'organization_name' => 1,
4490           'deceased_date' => 1,
4491           'is_deceased' => 1,
4492           'job_title' => 1,
4493           'legal_name' => 1,
4494           'sic_code' => 1,
4495           'current_employer' => 1,
4496           // FIXME: should we use defaultHierReturnProperties() for the below?
4497           'do_not_email' => 1,
4498           'do_not_mail' => 1,
4499           'do_not_sms' => 1,
4500           'do_not_phone' => 1,
4501           'do_not_trade' => 1,
4502           'is_opt_out' => 1,
4503           'contact_is_deleted' => 1,
4504           'preferred_communication_method' => 1,
4505           'preferred_language' => 1,
4506         ];
4507       }
4508     }
4509     return self::$_defaultReturnProperties[$mode];
4510   }
4511
4512   /**
4513    * Get primary condition for a sql clause.
4514    *
4515    * @param int $value
4516    *
4517    * @return string|NULL
4518    */
4519   public static function getPrimaryCondition($value) {
4520     if (is_numeric($value)) {
4521       $value = (int ) $value;
4522       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4523     }
4524     return NULL;
4525   }
4526
4527   /**
4528    * Wrapper for a simple search query.
4529    *
4530    * @param array $params
4531    * @param array $returnProperties
4532    * @param bool $count
4533    *
4534    * @return string
4535    * @throws \CRM_Core_Exception
4536    */
4537   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4538     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4539     list($select, $from, $where, $having) = $query->query();
4540     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4541
4542     $query = "$select $from $where $groupBy $having";
4543     return $query;
4544   }
4545
4546   /**
4547    * These are stub comments as this function needs more explanation - particularly in terms of how it
4548    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4549    *
4550    * This function was originally written as a wrapper for the api query but is called from multiple places
4551    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4552    * but it is unclear as to whehter that is historical or there is a reason
4553    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4554    *
4555    * @param array $params
4556    * @param array $returnProperties
4557    * @param null $fields
4558    * @param string $sort
4559    * @param int $offset
4560    * @param int $row_count
4561    * @param bool $smartGroupCache
4562    *   ?? update smart group cache?.
4563    * @param bool $count
4564    *   Return count obnly.
4565    * @param bool $skipPermissions
4566    *   Should permissions be ignored or should the logged in user's permissions be applied.
4567    * @param int $mode
4568    *   This basically correlates to the component.
4569    * @param string $apiEntity
4570    *   The api entity being called.
4571    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4572    *
4573    * @param bool|null $primaryLocationOnly
4574    *
4575    * @return array
4576    * @throws \CRM_Core_Exception
4577    */
4578   public static function apiQuery(
4579     $params = NULL,
4580     $returnProperties = NULL,
4581     $fields = NULL,
4582     $sort = NULL,
4583     $offset = 0,
4584     $row_count = 25,
4585     $smartGroupCache = TRUE,
4586     $count = FALSE,
4587     $skipPermissions = TRUE,
4588     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4589     $apiEntity = NULL,
4590     $primaryLocationOnly = NULL
4591   ) {
4592
4593     $query = new CRM_Contact_BAO_Query(
4594       $params, $returnProperties,
4595       NULL, TRUE, FALSE, $mode,
4596       $skipPermissions,
4597       TRUE, $smartGroupCache,
4598       NULL, 'AND',
4599       $apiEntity, $primaryLocationOnly
4600     );
4601
4602     //this should add a check for view deleted if permissions are enabled
4603     if ($skipPermissions) {
4604       $query->_skipDeleteClause = TRUE;
4605     }
4606     $query->generatePermissionClause(FALSE, $count);
4607
4608     // note : this modifies _fromClause and _simpleFromClause
4609     $query->includePseudoFieldsJoin($sort);
4610
4611     list($select, $from, $where, $having) = $query->query($count);
4612
4613     if (!empty($query->_permissionWhereClause)) {
4614       if (!empty($query->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
4615         $from .= " $query->_permissionFromClause";
4616       }
4617       if (empty($where)) {
4618         $where = "WHERE $query->_permissionWhereClause";
4619       }
4620       else {
4621         $where = "$where AND $query->_permissionWhereClause";
4622       }
4623     }
4624
4625     $sql = "$select $from $where $having";
4626
4627     // add group by only when API action is not getcount
4628     //  otherwise query fetches incorrect count
4629     if ($query->_useGroupBy && !$count) {
4630       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4631     }
4632     if (!empty($sort)) {
4633       $sort = CRM_Utils_Type::escape($sort, 'String');
4634       $sql .= " ORDER BY $sort ";
4635     }
4636     if ($row_count > 0 && $offset >= 0) {
4637       $offset = CRM_Utils_Type::escape($offset, 'Int');
4638       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4639       $sql .= " LIMIT $offset, $row_count ";
4640     }
4641
4642     $dao = CRM_Core_DAO::executeQuery($sql);
4643
4644     // @todo derive this from the component class rather than hard-code two options.
4645     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4646
4647     $values = [];
4648     while ($dao->fetch()) {
4649       if ($count) {
4650         $noRows = $dao->rowCount;
4651         return [$noRows, NULL];
4652       }
4653       $val = $query->store($dao);
4654       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4655
4656       if (!empty($convertedVals)) {
4657         $val = array_replace_recursive($val, $convertedVals);
4658       }
4659       $values[$dao->$entityIDField] = $val;
4660     }
4661     return [$values];
4662   }
4663
4664   /**
4665    * Get the actual custom field name by stripping off the appended string.
4666    *
4667    * The string could be _relative, _from, or _to
4668    *
4669    * @todo use metadata rather than convention to do this.
4670    *
4671    * @param string $parameterName
4672    *   The name of the parameter submitted to the form.
4673    *   e.g
4674    *   custom_3_relative
4675    *   custom_3_from
4676    *
4677    * @return string
4678    */
4679   public static function getCustomFieldName($parameterName) {
4680     if (substr($parameterName, -5, 5) == '_from') {
4681       return substr($parameterName, 0, strpos($parameterName, '_from'));
4682     }
4683     if (substr($parameterName, -9, 9) == '_relative') {
4684       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4685     }
4686     if (substr($parameterName, -3, 3) == '_to') {
4687       return substr($parameterName, 0, strpos($parameterName, '_to'));
4688     }
4689   }
4690
4691   /**
4692    * Convert submitted values for relative custom fields to query object format.
4693    *
4694    * The query will support the sqlOperator format so convert to that format.
4695    *
4696    * @param array $formValues
4697    *   Submitted values.
4698    * @param array $params
4699    *   Converted parameters for the query object.
4700    * @param string $values
4701    *   Submitted value.
4702    * @param string $fieldName
4703    *   Submitted field name. (Matches form field not DB field.)
4704    */
4705   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4706     if (empty($values)) {
4707       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4708       // this function gets called again after they fields have been converted which can get ugly.
4709       return;
4710     }
4711     $customFieldName = self::getCustomFieldName($fieldName);
4712
4713     if (substr($fieldName, -9, 9) == '_relative') {
4714       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4715     }
4716     else {
4717       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4718         // Both to & from are set. We only need to acton one, choosing from.
4719         return;
4720       }
4721
4722       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4723       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4724
4725       if (self::isCustomDateField($customFieldName)) {
4726         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4727       }
4728     }
4729
4730     if ($from) {
4731       if ($to) {
4732         $relativeFunction = ['BETWEEN' => [$from, $to]];
4733       }
4734       else {
4735         $relativeFunction = ['>=' => $from];
4736       }
4737     }
4738     else {
4739       $relativeFunction = ['<=' => $to];
4740     }
4741     $params[] = [
4742       $customFieldName,
4743       '=',
4744       $relativeFunction,
4745       0,
4746       0,
4747     ];
4748   }
4749
4750   /**
4751    * Are we dealing with custom field of type date.
4752    *
4753    * @param $fieldName
4754    *
4755    * @return bool
4756    * @throws \CiviCRM_API3_Exception
4757    */
4758   public static function isCustomDateField($fieldName) {
4759     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4760       return FALSE;
4761     }
4762     try {
4763       $customFieldDataType = civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type']);
4764       if ('Date' == $customFieldDataType) {
4765         return TRUE;
4766       }
4767     }
4768     catch (CiviCRM_API3_Exception $e) {
4769     }
4770     return FALSE;
4771   }
4772
4773   /**
4774    * Has this field already been reformatting to Query object syntax.
4775    *
4776    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4777    * to come with associated double queries & is possibly damaging performance.
4778    *
4779    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4780    * them as they are.
4781    *
4782    * @param mixed $values
4783    *   Value in formValues for the field.
4784    *
4785    * @return bool;
4786    */
4787   public static function isAlreadyProcessedForQueryFormat($values) {
4788     if (!is_array($values)) {
4789       return FALSE;
4790     }
4791     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4792       return FALSE;
4793     }
4794     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4795   }
4796
4797   /**
4798    * If the state and country are passed remove state.
4799    *
4800    * Country is implicit from the state, but including both results in
4801    * a poor query as there is no combined index on state AND country.
4802    *
4803    * @see https://issues.civicrm.org/jira/browse/CRM-18125
4804    *
4805    * @param array $formValues
4806    */
4807   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4808     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4809       // The use of array map sanitises the data by ensuring we are dealing with integers.
4810       $states = implode(', ', array_map('intval', $formValues['state_province']));
4811       $countryList = CRM_Core_DAO::singleValueQuery(
4812         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4813       );
4814       if ($countryList == $formValues['country']) {
4815         unset($formValues['country']);
4816       }
4817     }
4818   }
4819
4820   /**
4821    * For some special cases, grouping by subset of select fields becomes mandatory.
4822    * Hence, full_group_by mode is handled by appending any_value
4823    * keyword to select fields not present in groupBy
4824    *
4825    * @param array $selectClauses
4826    * @param array $groupBy - Columns already included in GROUP By clause.
4827    * @param string $aggregateFunction
4828    *
4829    * @return string
4830    */
4831   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4832     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4833       $groupBy = array_map('trim', (array) $groupBy);
4834       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4835       foreach ($selectClauses as $key => &$val) {
4836         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4837         // append ANY_VALUE() keyword
4838         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4839           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4840             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4841             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4842         }
4843       }
4844     }
4845
4846     return "SELECT " . implode(', ', $selectClauses) . " ";
4847   }
4848
4849   /**
4850    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4851    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4852    * keyword to select fields not present in groupBy
4853    *
4854    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4855    * @param array $orderBys - ORDER BY sub-clauses
4856    *
4857    */
4858   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4859     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4860       foreach ($orderBys as $orderBy) {
4861         // remove sort syntax from ORDER BY clauses if present
4862         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4863         // if ORDER BY column is not present in GROUP BY then append it to end
4864         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4865           $groupBy .= ", {$orderBy}";
4866         }
4867       }
4868     }
4869   }
4870
4871   /**
4872    * Include Select columns in groupBy clause.
4873    *
4874    * @param array $selectClauses
4875    * @param array $groupBy - Columns already included in GROUP By clause.
4876    *
4877    * @return string
4878    */
4879   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4880     $groupBy = (array) $groupBy;
4881     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4882     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4883
4884     //return if ONLY_FULL_GROUP_BY is not enabled.
4885     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4886       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4887       foreach ($selectClauses as $key => $val) {
4888         $aliasArray = preg_split('/ as /i', $val);
4889         // if more than 1 alias we need to split by ','.
4890         if (count($aliasArray) > 2) {
4891           $aliasArray = preg_split('/,/', $val);
4892           foreach ($aliasArray as $key => $value) {
4893             $alias = current(preg_split('/ as /i', $value));
4894             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4895               $groupBy[] = $alias;
4896             }
4897           }
4898         }
4899         else {
4900           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4901           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4902           $tableName = current(explode('.', $selectColumn));
4903           $primaryKey = "{$tableName}.id";
4904           // exclude columns which are already included in groupBy and aggregate functions from select
4905           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4906           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4907             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4908               $groupBy[] = $alias;
4909             }
4910             else {
4911               $groupBy[] = $selectColumn;
4912             }
4913           }
4914         }
4915       }
4916     }
4917
4918     if (!empty($groupBy)) {
4919       return " GROUP BY " . implode(', ', $groupBy);
4920     }
4921     return '';
4922   }
4923
4924   /**
4925    * Create and query the db for an contact search.
4926    *
4927    * @param int $offset
4928    *   The offset for the query.
4929    * @param int $rowCount
4930    *   The number of rows to return.
4931    * @param string|CRM_Utils_Sort $sort
4932    *   The order by string.
4933    * @param bool $count
4934    *   Is this a count only query ?.
4935    * @param bool $includeContactIds
4936    *   Should we include contact ids?.
4937    * @param bool $sortByChar
4938    *   If true returns the distinct array of first characters for search results.
4939    * @param bool $groupContacts
4940    *   If true, return only the contact ids.
4941    * @param bool $returnQuery
4942    *   Should we return the query as a string.
4943    * @param string $additionalWhereClause
4944    *   If the caller wants to further restrict the search (used for components).
4945    * @param null $sortOrder
4946    * @param string $additionalFromClause
4947    *   Should be clause with proper joins, effective to reduce where clause load.
4948    *
4949    * @param bool $skipOrderAndLimit
4950    *
4951    * @return CRM_Core_DAO
4952    */
4953   public function searchQuery(
4954     $offset = 0, $rowCount = 0, $sort = NULL,
4955     $count = FALSE, $includeContactIds = FALSE,
4956     $sortByChar = FALSE, $groupContacts = FALSE,
4957     $returnQuery = FALSE,
4958     $additionalWhereClause = NULL, $sortOrder = NULL,
4959     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4960   ) {
4961
4962     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4963
4964     if ($returnQuery) {
4965       return $query;
4966     }
4967     if ($count) {
4968       return CRM_Core_DAO::singleValueQuery($query);
4969     }
4970
4971     $dao = CRM_Core_DAO::executeQuery($query);
4972
4973     // We can always call this - it will only re-enable if it was originally enabled.
4974     CRM_Core_DAO::reenableFullGroupByMode();
4975
4976     if ($groupContacts) {
4977       $ids = [];
4978       while ($dao->fetch()) {
4979         $ids[] = $dao->id;
4980       }
4981       return implode(',', $ids);
4982     }
4983
4984     return $dao;
4985   }
4986
4987   /**
4988    * Create and query the db for the list of all first letters used by contacts
4989    *
4990    * @return CRM_Core_DAO
4991    */
4992   public function alphabetQuery() {
4993     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
4994     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
4995       {$sqlParts['from']}
4996       {$sqlParts['where']}";
4997     $dao = CRM_Core_DAO::executeQuery($query);
4998     return $dao;
4999   }
5000
5001   /**
5002    * Fetch a list of contacts for displaying a search results page
5003    *
5004    * @param array $cids
5005    *   List of contact IDs
5006    * @param bool $includeContactIds
5007    * @return CRM_Core_DAO
5008    */
5009   public function getCachedContacts($cids, $includeContactIds) {
5010     CRM_Core_DAO::disableFullGroupByMode();
5011     CRM_Utils_Type::validateAll($cids, 'Positive');
5012     $this->_includeContactIds = $includeContactIds;
5013     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5014     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5015     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5016     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5017     $order = 'ORDER BY _wgt';
5018     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5019     $limit = '';
5020     $query = "$select $from $where $groupBy $order $limit";
5021
5022     $result = CRM_Core_DAO::executeQuery($query);
5023     CRM_Core_DAO::reenableFullGroupByMode();
5024     return $result;
5025   }
5026
5027   /**
5028    * Construct a SQL CASE expression.
5029    *
5030    * @param string $idCol
5031    *   The name of a column with ID's (eg 'contact_a.id').
5032    * @param array $cids
5033    *   Array(int $weight => int $id).
5034    * @return string
5035    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5036    */
5037   private function createSqlCase($idCol, $cids) {
5038     $buf = "CASE\n";
5039     foreach ($cids as $weight => $cid) {
5040       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5041     }
5042     $buf .= "END\n";
5043     return $buf;
5044   }
5045
5046   /**
5047    * Populate $this->_permissionWhereClause with permission related clause and update other
5048    * query related properties.
5049    *
5050    * Function calls ACL permission class and hooks to filter the query appropriately
5051    *
5052    * Note that these 2 params were in the code when extracted from another function
5053    * and a second round extraction would be to make them properties of the class
5054    *
5055    * @param bool $onlyDeleted
5056    *   Only get deleted contacts.
5057    * @param bool $count
5058    *   Return Count only.
5059    */
5060   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5061     if (!$this->_skipPermission) {
5062       $permissionClauses = CRM_Contact_BAO_Contact_Permission::cacheClause();
5063       $this->_permissionWhereClause = $permissionClauses[1];
5064       $this->_permissionFromClause = $permissionClauses[0];
5065
5066       if (CRM_Core_Permission::check('access deleted contacts')) {
5067         if (!$onlyDeleted) {
5068           $this->_permissionWhereClause .= ' AND (contact_a.is_deleted = 0)';
5069         }
5070         else {
5071           $this->_permissionWhereClause .= " AND (contact_a.is_deleted) ";
5072         }
5073       }
5074
5075       if (isset($this->_tables['civicrm_activity'])) {
5076         $bao = new CRM_Activity_BAO_Activity();
5077         $clauses = $subclauses = [];
5078         foreach ((array) $bao->addSelectWhereClause() as $field => $vals) {
5079           if ($vals && $field !== 'id') {
5080             $clauses[] = $bao->tableName() . ".$field " . $vals;
5081           }
5082           elseif ($vals) {
5083             $subclauses[] = "$field " . implode(" AND $field ", (array) $vals);
5084           }
5085         }
5086         if ($subclauses) {
5087           $clauses[] = $bao->tableName() . '.`id` IN (SELECT `id` FROM `' . $bao->tableName() . '` WHERE ' . implode(' AND ', $subclauses) . ')';
5088         }
5089         if (!empty($clauses) && $this->_permissionWhereClause) {
5090           $this->_permissionWhereClause .= ' AND (' . implode(' AND ', $clauses) . ')';
5091         }
5092         elseif (!empty($clauses)) {
5093           $this->_permissionWhereClause .= '(' . implode(' AND ', $clauses) . ')';
5094         }
5095       }
5096     }
5097     else {
5098       // add delete clause if needed even if we are skipping permission
5099       // CRM-7639
5100       if (!$this->_skipDeleteClause) {
5101         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5102           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5103         }
5104         else {
5105           // CRM-6181
5106           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5107         }
5108       }
5109     }
5110   }
5111
5112   /**
5113    * @param $val
5114    */
5115   public function setSkipPermission($val) {
5116     $this->_skipPermission = $val;
5117   }
5118
5119   /**
5120    * @param null $context
5121    *
5122    * @return array
5123    * @throws \CRM_Core_Exception
5124    */
5125   public function summaryContribution($context = NULL) {
5126     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5127     if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5128       $from .= " $this->_permissionFromClause";
5129     }
5130     if ($this->_permissionWhereClause) {
5131       $where .= " AND " . $this->_permissionWhereClause;
5132     }
5133     if ($context == 'search') {
5134       $where .= " AND contact_a.is_deleted = 0 ";
5135     }
5136
5137     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5138
5139     $summary = ['total' => []];
5140     $this->addBasicStatsToSummary($summary, $where, $from);
5141
5142     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5143       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5144     }
5145
5146     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5147
5148     return $summary;
5149   }
5150
5151   /**
5152    * Append financial ACL limits to the query from & where clauses, if applicable.
5153    *
5154    * @param string $where
5155    * @param string $from
5156    */
5157   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5158     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5159       return;
5160     }
5161     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5162     if (!empty($financialTypes)) {
5163       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5164       $from .= " LEFT JOIN civicrm_line_item li
5165                   ON civicrm_contribution.id = li.contribution_id AND
5166                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5167     }
5168     else {
5169       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5170     }
5171   }
5172
5173   /**
5174    * Getter for the qill object.
5175    *
5176    * @return array
5177    */
5178   public function qill() {
5179     return $this->_qill;
5180   }
5181
5182   /**
5183    * Default set of return default hier return properties.
5184    *
5185    * @return array
5186    */
5187   public static function &defaultHierReturnProperties() {
5188     if (!isset(self::$_defaultHierReturnProperties)) {
5189       self::$_defaultHierReturnProperties = [
5190         'home_URL' => 1,
5191         'image_URL' => 1,
5192         'legal_identifier' => 1,
5193         'external_identifier' => 1,
5194         'contact_type' => 1,
5195         'contact_sub_type' => 1,
5196         'sort_name' => 1,
5197         'display_name' => 1,
5198         'nick_name' => 1,
5199         'first_name' => 1,
5200         'middle_name' => 1,
5201         'last_name' => 1,
5202         'prefix_id' => 1,
5203         'suffix_id' => 1,
5204         'formal_title' => 1,
5205         'communication_style_id' => 1,
5206         'email_greeting' => 1,
5207         'postal_greeting' => 1,
5208         'addressee' => 1,
5209         'birth_date' => 1,
5210         'gender_id' => 1,
5211         'preferred_communication_method' => 1,
5212         'do_not_phone' => 1,
5213         'do_not_email' => 1,
5214         'do_not_mail' => 1,
5215         'do_not_sms' => 1,
5216         'do_not_trade' => 1,
5217         'location' => [
5218           '1' => [
5219             'location_type' => 1,
5220             'street_address' => 1,
5221             'city' => 1,
5222             'state_province' => 1,
5223             'postal_code' => 1,
5224             'postal_code_suffix' => 1,
5225             'country' => 1,
5226             'phone-Phone' => 1,
5227             'phone-Mobile' => 1,
5228             'phone-Fax' => 1,
5229             'phone-1' => 1,
5230             'phone-2' => 1,
5231             'phone-3' => 1,
5232             'im-1' => 1,
5233             'im-2' => 1,
5234             'im-3' => 1,
5235             'email-1' => 1,
5236             'email-2' => 1,
5237             'email-3' => 1,
5238           ],
5239           '2' => [
5240             'location_type' => 1,
5241             'street_address' => 1,
5242             'city' => 1,
5243             'state_province' => 1,
5244             'postal_code' => 1,
5245             'postal_code_suffix' => 1,
5246             'country' => 1,
5247             'phone-Phone' => 1,
5248             'phone-Mobile' => 1,
5249             'phone-1' => 1,
5250             'phone-2' => 1,
5251             'phone-3' => 1,
5252             'im-1' => 1,
5253             'im-2' => 1,
5254             'im-3' => 1,
5255             'email-1' => 1,
5256             'email-2' => 1,
5257             'email-3' => 1,
5258           ],
5259         ],
5260       ];
5261     }
5262     return self::$_defaultHierReturnProperties;
5263   }
5264
5265   /**
5266    * Build query for a date field.
5267    *
5268    * @param array $values
5269    * @param string $tableName
5270    * @param string $fieldName
5271    * @param string $dbFieldName
5272    * @param string $fieldTitle
5273    * @param bool $appendTimeStamp
5274    * @param string $dateFormat
5275    * @param string|null $highDBFieldName
5276    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5277    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5278    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5279    *   date & less than the end date.
5280    *
5281    * @throws \CRM_Core_Exception
5282    */
5283   public function dateQueryBuilder(
5284     $values, $tableName, $fieldName,
5285     $dbFieldName, $fieldTitle,
5286     $appendTimeStamp = TRUE,
5287     $dateFormat = 'YmdHis',
5288     $highDBFieldName = NULL
5289   ) {
5290     // @todo - remove dateFormat - pretty sure it's never passed in...
5291     list($name, $op, $value, $grouping, $wildcard) = $values;
5292     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5293       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5294       return;
5295     }
5296     if ($tableName === 'civicrm_contact') {
5297       // Special handling for contact table as it has a known alias in advanced search.
5298       $tableName = 'contact_a';
5299     }
5300     if ($name === "{$fieldName}_low" ||
5301       $name === "{$fieldName}_high"
5302     ) {
5303       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5304         return;
5305       }
5306       $this->_rangeCache[$fieldName] = 1;
5307
5308       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5309
5310       if ($name == $fieldName . '_low') {
5311         $firstOP = '>=';
5312         $firstPhrase = ts('greater than or equal to');
5313         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5314
5315         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5316         if (!empty($secondValues) && $secondValues[2]) {
5317           $secondOP = '<=';
5318           $secondPhrase = ts('less than or equal to');
5319           $secondValue = $secondValues[2];
5320
5321           if ($appendTimeStamp && strlen($secondValue) == 10) {
5322             $secondValue .= ' 23:59:59';
5323           }
5324           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5325         }
5326       }
5327       elseif ($name == $fieldName . '_high') {
5328         $firstOP = '<=';
5329         $firstPhrase = ts('less than or equal to');
5330
5331         if ($appendTimeStamp && strlen($value) == 10) {
5332           $value .= ' 23:59:59';
5333         }
5334         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5335
5336         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5337         if (!empty($secondValues) && $secondValues[2]) {
5338           $secondOP = '>=';
5339           $secondPhrase = ts('greater than or equal to');
5340           $secondValue = $secondValues[2];
5341           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5342         }
5343       }
5344
5345       if (!$appendTimeStamp) {
5346         $firstDate = substr($firstDate, 0, 8);
5347       }
5348       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5349
5350       if ($secondDate) {
5351         if (!$appendTimeStamp) {
5352           $secondDate = substr($secondDate, 0, 8);
5353         }
5354         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5355       }
5356
5357       if ($secondDate) {
5358         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5359         $this->_where[$grouping][] = "
5360 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5361 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5362 ";
5363         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5364       }
5365       else {
5366         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5367         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5368       }
5369     }
5370
5371     if ($name == $fieldName) {
5372       //In Get API, for operators other then '=' the $value is in array(op => value) format
5373       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5374         $op = key($value);
5375         $value = $value[$op];
5376       }
5377
5378       $date = $format = NULL;
5379       if (strstr($op, 'IN')) {
5380         $format = [];
5381         foreach ($value as &$date) {
5382           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5383           if (!$appendTimeStamp) {
5384             $date = substr($date, 0, 8);
5385           }
5386           $format[] = CRM_Utils_Date::customFormat($date);
5387         }
5388         $date = "('" . implode("','", $value) . "')";
5389         $format = implode(', ', $format);
5390       }
5391       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5392         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5393         if (!$appendTimeStamp) {
5394           $date = substr($date, 0, 8);
5395         }
5396         $format = CRM_Utils_Date::customFormat($date);
5397         $date = "'$date'";
5398       }
5399
5400       if ($date) {
5401         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5402       }
5403       else {
5404         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5405       }
5406
5407       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5408       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5409     }
5410
5411     // Ensure the tables are set, but don't whomp anything.
5412     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5413     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5414   }
5415
5416   /**
5417    * @param $values
5418    * @param string $tableName
5419    * @param string $fieldName
5420    * @param string $dbFieldName
5421    * @param $fieldTitle
5422    * @param null $options
5423    */
5424   public function numberRangeBuilder(
5425     &$values,
5426     $tableName, $fieldName,
5427     $dbFieldName, $fieldTitle,
5428     $options = NULL
5429   ) {
5430     list($name, $op, $value, $grouping, $wildcard) = $values;
5431
5432     if ($name == "{$fieldName}_low" ||
5433       $name == "{$fieldName}_high"
5434     ) {
5435       if (isset($this->_rangeCache[$fieldName])) {
5436         return;
5437       }
5438       $this->_rangeCache[$fieldName] = 1;
5439
5440       $secondOP = $secondPhrase = $secondValue = NULL;
5441
5442       if ($name == "{$fieldName}_low") {
5443         $firstOP = '>=';
5444         $firstPhrase = ts('greater than');
5445
5446         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5447         if (!empty($secondValues)) {
5448           $secondOP = '<=';
5449           $secondPhrase = ts('less than');
5450           $secondValue = $secondValues[2];
5451         }
5452       }
5453       else {
5454         $firstOP = '<=';
5455         $firstPhrase = ts('less than');
5456
5457         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5458         if (!empty($secondValues)) {
5459           $secondOP = '>=';
5460           $secondPhrase = ts('greater than');
5461           $secondValue = $secondValues[2];
5462         }
5463       }
5464
5465       if ($secondOP) {
5466         $this->_where[$grouping][] = "
5467 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5468 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5469 ";
5470         $displayValue = $options ? $options[$value] : $value;
5471         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5472
5473         $this->_qill[$grouping][]
5474           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5475       }
5476       else {
5477         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5478         $displayValue = $options ? $options[$value] : $value;
5479         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5480       }
5481       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5482
5483       return;
5484     }
5485
5486     if ($name == $fieldName) {
5487       $op = '=';
5488       $phrase = '=';
5489
5490       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5491
5492       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5493       $displayValue = $options ? $options[$value] : $value;
5494       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5495     }
5496   }
5497
5498   /**
5499    * @param $values
5500    * @param string $tableName
5501    * @param string $fieldName
5502    * @param string $dbFieldName
5503    * @param $fieldTitle
5504    * @param null $options
5505    */
5506   public function ageRangeQueryBuilder(
5507     &$values,
5508     $tableName, $fieldName,
5509     $dbFieldName, $fieldTitle,
5510     $options = NULL
5511   ) {
5512     list($name, $op, $value, $grouping, $wildcard) = $values;
5513
5514     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5515     // will be treated as current day
5516     $asofDate = NULL;
5517     if ($asofDateValues) {
5518       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5519       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5520       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5521     }
5522
5523     if ($name == "{$fieldName}_low" ||
5524       $name == "{$fieldName}_high"
5525     ) {
5526       if (isset($this->_rangeCache[$fieldName])) {
5527         return;
5528       }
5529       $this->_rangeCache[$fieldName] = 1;
5530
5531       $secondOP = $secondPhrase = $secondValue = NULL;
5532
5533       if ($name == "{$fieldName}_low") {
5534         $firstPhrase = ts('greater than or equal to');
5535         // NB: age > X means date of birth < Y
5536         $firstOP = '<=';
5537         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5538
5539         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5540         if (!empty($secondValues)) {
5541           $secondOP = '>=';
5542           $secondPhrase = ts('less than or equal to');
5543           $secondValue = $secondValues[2];
5544           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5545         }
5546       }
5547       else {
5548         $firstOP = '>=';
5549         $firstPhrase = ts('less than or equal to');
5550         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5551
5552         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5553         if (!empty($secondValues)) {
5554           $secondOP = '<=';
5555           $secondPhrase = ts('greater than or equal to');
5556           $secondValue = $secondValues[2];
5557           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5558         }
5559       }
5560
5561       if ($secondOP) {
5562         $this->_where[$grouping][] = "
5563 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5564 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5565 ";
5566         $displayValue = $options ? $options[$value] : $value;
5567         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5568
5569         $this->_qill[$grouping][]
5570           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5571       }
5572       else {
5573         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5574         $displayValue = $options ? $options[$value] : $value;
5575         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5576       }
5577       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5578       return;
5579     }
5580   }
5581
5582   /**
5583    * Calculate date from age.
5584    *
5585    * @param string $asofDate
5586    * @param int $age
5587    * @param string $type
5588    *
5589    * @return string
5590    * @throws \Exception
5591    */
5592   public static function calcDateFromAge($asofDate, $age, $type) {
5593     $date = new DateTime($asofDate);
5594     if ($type == "min") {
5595       // minimum age is $age: dob <= date - age "235959"
5596       $date->sub(new DateInterval("P" . $age . "Y"));
5597       return $date->format('Ymd') . "235959";
5598     }
5599     else {
5600       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5601       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5602       return $date->format('Ymd') . "000000";
5603     }
5604   }
5605
5606   /**
5607    * Given the field name, operator, value & its data type
5608    * builds the where Clause for the query
5609    * used for handling 'IS NULL'/'IS NOT NULL' operators
5610    *
5611    * @param string $field
5612    *   Fieldname.
5613    * @param string $op
5614    *   Operator.
5615    * @param string $value
5616    *   Value.
5617    * @param string $dataType
5618    *   Data type of the field.
5619    *
5620    * @return string
5621    *   Where clause for the query.
5622    * @throws \CRM_Core_Exception
5623    */
5624   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5625     $op = trim($op);
5626     $clause = "$field $op";
5627
5628     switch ($op) {
5629       case 'IS NULL':
5630       case 'IS NOT NULL':
5631         return $clause;
5632
5633       case 'IS EMPTY':
5634         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5635         return $clause;
5636
5637       case 'IS NOT EMPTY':
5638         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5639         return $clause;
5640
5641       case 'RLIKE':
5642         return " CAST({$field} AS BINARY) RLIKE BINARY '{$value}' ";
5643
5644       case 'IN':
5645       case 'NOT IN':
5646         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5647         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5648           $value = [$op => (array) $value];
5649         }
5650
5651       default:
5652         if (empty($dataType) || $dataType == 'Date') {
5653           $dataType = 'String';
5654         }
5655         if (is_array($value)) {
5656           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5657           // widely used and consistent across the codebase
5658           // adding this here won't accept the search functions which don't submit an array
5659           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5660
5661             return $queryString;
5662           }
5663           if (!empty($value[0]) && $op === 'BETWEEN') {
5664             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5665             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5666               return $queryString;
5667             }
5668           }
5669           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5670         }
5671
5672         $value = CRM_Utils_Type::escape($value, $dataType);
5673         // if we don't have a dataType we should assume
5674         if ($dataType == 'String' || $dataType == 'Text') {
5675           $value = "'" . $value . "'";
5676         }
5677         return "$clause $value";
5678     }
5679   }
5680
5681   /**
5682    * @param bool $reset
5683    *
5684    * @return array
5685    */
5686   public function openedSearchPanes($reset = FALSE) {
5687     if (!$reset || empty($this->_whereTables)) {
5688       return self::$_openedPanes;
5689     }
5690
5691     // pane name to table mapper
5692     $panesMapper = [
5693       ts('Contributions') => 'civicrm_contribution',
5694       ts('Memberships') => 'civicrm_membership',
5695       ts('Events') => 'civicrm_participant',
5696       ts('Relationships') => 'civicrm_relationship',
5697       ts('Activities') => 'civicrm_activity',
5698       ts('Pledges') => 'civicrm_pledge',
5699       ts('Cases') => 'civicrm_case',
5700       ts('Grants') => 'civicrm_grant',
5701       ts('Address Fields') => 'civicrm_address',
5702       ts('Notes') => 'civicrm_note',
5703       ts('Change Log') => 'civicrm_log',
5704       ts('Mailings') => 'civicrm_mailing',
5705     ];
5706     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5707
5708     foreach (array_keys($this->_whereTables) as $table) {
5709       if ($panName = array_search($table, $panesMapper)) {
5710         self::$_openedPanes[$panName] = TRUE;
5711       }
5712     }
5713
5714     return self::$_openedPanes;
5715   }
5716
5717   /**
5718    * @param $operator
5719    */
5720   public function setOperator($operator) {
5721     $validOperators = ['AND', 'OR'];
5722     if (!in_array($operator, $validOperators)) {
5723       $operator = 'AND';
5724     }
5725     $this->_operator = $operator;
5726   }
5727
5728   /**
5729    * @return string
5730    */
5731   public function getOperator() {
5732     return $this->_operator;
5733   }
5734
5735   /**
5736    * @param $from
5737    * @param $where
5738    * @param $having
5739    */
5740   public function filterRelatedContacts(&$from, &$where, &$having) {
5741     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5742       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5743     }
5744     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5745     // since there only can be one instance of this filter in every query
5746     // skip if filter has already applied
5747     foreach ($_rTempCache as $acache) {
5748       foreach ($acache['queries'] as $aqcache) {
5749         if (strpos($from, $aqcache['from']) !== FALSE) {
5750           $having = NULL;
5751           return;
5752         }
5753       }
5754     }
5755     $arg_sig = sha1("$from $where $having");
5756     if (isset($_rTempCache[$arg_sig])) {
5757       $cache = $_rTempCache[$arg_sig];
5758     }
5759     else {
5760       // create temp table with contact ids
5761
5762       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5763
5764       $sql = "
5765 REPLACE INTO $tableName ( contact_id )
5766 SELECT contact_a.id
5767        $from
5768        $where
5769        $having
5770 ";
5771       CRM_Core_DAO::executeQuery($sql);
5772
5773       $cache = ['tableName' => $tableName, 'queries' => []];
5774       $_rTempCache[$arg_sig] = $cache;
5775     }
5776     // upsert the query depending on relationship type
5777     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5778       $qcache = $cache['queries'][$this->_displayRelationshipType];
5779     }
5780     else {
5781       $tableName = $cache['tableName'];
5782       $qcache = [
5783         "from" => "",
5784         "where" => "",
5785       ];
5786       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5787       if (is_numeric($this->_displayRelationshipType)) {
5788         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5789         $qcache['from'] = "
5790 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5791 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5792 ";
5793         $qcache['where'] = "
5794 AND displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5795 AND   displayRelType.is_active = 1
5796 ";
5797       }
5798       else {
5799         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5800         if ($dirOne == 'a') {
5801           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5802           $qcache['from'] .= "
5803 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5804 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5805 ";
5806         }
5807         else {
5808           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5809           $qcache['from'] .= "
5810 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5811 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5812 ";
5813         }
5814         $qcache['where'] = "
5815 AND displayRelType.relationship_type_id = $relType
5816 AND   displayRelType.is_active = 1
5817 ";
5818       }
5819       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5820       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5821     }
5822     $qillMessage = ts('Contacts with a Relationship Type of: ');
5823     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5824     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5825       $this->_qill[0][] = $iqill;
5826     }
5827     if (strpos($from, $qcache['from']) === FALSE) {
5828       if (strpos($from, "INNER JOIN") !== FALSE) {
5829         // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5830         // this happens when we have an event_type in the quert (CRM-7969)
5831         $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5832         // Make sure the relationship join right after the FROM and other joins afterwards.
5833         // This gives us the possibility to change the join on civicrm case.
5834         $from = preg_replace("/LEFT JOIN/", $qcache['from'] . " LEFT JOIN", $from, 1);
5835       }
5836       else {
5837         $from .= $qcache['from'];
5838       }
5839       if (!strlen($where)) {
5840         $where = " WHERE 1 ";
5841       }
5842       $where .= $qcache['where'];
5843       if (!empty($this->_tables['civicrm_case'])) {
5844         // Change the join on CiviCRM case so that it joins on the right contac from the relationship.
5845         $from = str_replace("ON civicrm_case_contact.contact_id = contact_a.id", "ON civicrm_case_contact.contact_id = transform_temp.contact_id", $from);
5846         $where = str_replace("AND civicrm_case_contact.contact_id = contact_a.id", "AND civicrm_case_contact.contact_id = transform_temp.contact_id", $where);
5847         $where .= " AND displayRelType.case_id = civicrm_case_contact.case_id ";
5848       }
5849       if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5850         $from .= " $this->_permissionFromClause";
5851       }
5852       if (!empty($this->_permissionWhereClause)) {
5853         $where .= "AND $this->_permissionWhereClause";
5854       }
5855     }
5856
5857     $having = NULL;
5858   }
5859
5860   /**
5861    * See CRM-19811 for why this is database hurty without apparent benefit.
5862    *
5863    * @param $op
5864    *
5865    * @return bool
5866    */
5867   public static function caseImportant($op) {
5868     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5869   }
5870
5871   /**
5872    * @param $returnProperties
5873    * @param $prefix
5874    *
5875    * @return bool
5876    */
5877   public static function componentPresent(&$returnProperties, $prefix) {
5878     foreach ($returnProperties as $name => $dontCare) {
5879       if (substr($name, 0, strlen($prefix)) == $prefix) {
5880         return TRUE;
5881       }
5882     }
5883     return FALSE;
5884   }
5885
5886   /**
5887    * Builds the necessary structures for all fields that are similar to option value look-ups.
5888    *
5889    * @param string $name
5890    *   the name of the field.
5891    * @param string $op
5892    *   the sql operator, this function should handle ALL SQL operators.
5893    * @param string $value
5894    *   depends on the operator and who's calling the query builder.
5895    * @param int $grouping
5896    *   the index where to place the where clause.
5897    * @param string $daoName
5898    *   DAO Name.
5899    * @param array $field
5900    *   an array that contains various properties of the field identified by $name.
5901    * @param string $label
5902    *   The label for this field element.
5903    * @param string $dataType
5904    *
5905    * @throws \CRM_Core_Exception
5906    */
5907   public function optionValueQuery(
5908     $name,
5909     $op,
5910     $value,
5911     $grouping,
5912     $daoName,
5913     $field,
5914     $label,
5915     $dataType = 'String'
5916   ) {
5917
5918     $pseudoFields = [
5919       'email_greeting',
5920       'postal_greeting',
5921       'addressee',
5922     ];
5923
5924     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5925     if ($tableName == 'civicrm_contact') {
5926       $wc = "contact_a.$fieldName";
5927     }
5928     else {
5929       // Special handling for on_hold, so that we actually use the 'where'
5930       // property in order to limit the query by the on_hold status of the email,
5931       // instead of using email.id which would be nonsensical.
5932       if ($field['name'] === 'on_hold') {
5933         $wc = $field['where'];
5934       }
5935       else {
5936         $wc = "$tableName.id";
5937       }
5938     }
5939
5940     if (in_array($name, $pseudoFields)) {
5941       $wc = "contact_a.{$name}_id";
5942       $dataType = 'Positive';
5943       $value = (!$value) ? 0 : $value;
5944     }
5945     if ($name == "world_region") {
5946       $field['name'] = $name;
5947     }
5948
5949     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5950     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5951     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5952   }
5953
5954   /**
5955    * Check and explode a user defined numeric string into an array
5956    * this was the protocol used by search builder in the old old days before we had
5957    * super nice js widgets to do the hard work
5958    *
5959    * @param string $string
5960    * @param string $dataType
5961    *   The dataType we should check for the values, default integer.
5962    *
5963    * @return bool|array
5964    *   false if string does not match the pattern
5965    *   array of numeric values if string does match the pattern
5966    */
5967   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5968     $string = trim($string);
5969     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5970       return FALSE;
5971     }
5972
5973     $string = substr($string, 1, -1);
5974     $values = explode(',', $string);
5975     if (empty($values)) {
5976       return FALSE;
5977     }
5978
5979     $returnValues = [];
5980     foreach ($values as $v) {
5981       if ($dataType == 'Integer' && !is_numeric($v)) {
5982         return FALSE;
5983       }
5984       elseif ($dataType == 'String' && !is_string($v)) {
5985         return FALSE;
5986       }
5987       $returnValues[] = trim($v);
5988     }
5989
5990     if (empty($returnValues)) {
5991       return FALSE;
5992     }
5993
5994     return $returnValues;
5995   }
5996
5997   /**
5998    * Convert the pseudo constants id's to their names
5999    *
6000    * @param CRM_Core_DAO $dao
6001    * @param bool $return
6002    * @param bool $usedForAPI
6003    *
6004    * @return array|NULL
6005    */
6006   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
6007     if (empty($this->_pseudoConstantsSelect)) {
6008       return NULL;
6009     }
6010     $values = [];
6011     foreach ($this->_pseudoConstantsSelect as $key => $value) {
6012       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
6013         continue;
6014       }
6015
6016       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6017         $val = $dao->{$value['idCol']};
6018         if ($key == 'groups') {
6019           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6020           continue;
6021         }
6022
6023         if (CRM_Utils_System::isNull($val)) {
6024           $dao->$key = NULL;
6025         }
6026         elseif (!empty($value['pseudoconstant'])) {
6027           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6028           // and we can use the metadata to figure it out.
6029           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6030           // more metadata based choices.
6031           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6032             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6033             foreach ($dbValues as $pseudoValue) {
6034               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6035             }
6036
6037             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6038             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
6039             if ($usedForAPI && $realFieldName) {
6040               // normally we would see 2 fields returned for pseudoConstants. An exception is
6041               // preferred_communication_method where there is no id-variant.
6042               // For the api we prioritise getting the real data returned.
6043               // over the resolved version
6044               $dao->$realFieldName = $dbValues;
6045             }
6046
6047           }
6048           else {
6049             // This is basically the same as the default but since we have the bao we can use
6050             // a cached function.
6051             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6052           }
6053         }
6054         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6055           //preserve id value
6056           $idColumn = "{$key}_id";
6057           $dao->$idColumn = $val;
6058
6059           if ($key == 'state_province_name') {
6060             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6061           }
6062           else {
6063             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6064           }
6065         }
6066         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6067           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6068         }
6069         // @todo handle this in the section above for pseudoconstants.
6070         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6071           // @todo define bao on this & merge into the above condition.
6072           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6073
6074           if ($value['pseudoField'] == 'participant_role') {
6075             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6076             foreach ($viewValues as $k => $v) {
6077               $viewValues[$k] = $pseudoOptions[$v];
6078             }
6079           }
6080           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6081         }
6082         else {
6083           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6084           $dao->$key = $labels[$val];
6085         }
6086
6087         // return converted values in array format
6088         if ($return) {
6089           if (strpos($key, '-') !== FALSE) {
6090             $keyVal = explode('-', $key);
6091             $current = &$values;
6092             $lastElement = array_pop($keyVal);
6093             foreach ($keyVal as $v) {
6094               if (!array_key_exists($v, $current)) {
6095                 $current[$v] = [];
6096               }
6097               $current = &$current[$v];
6098             }
6099             $current[$lastElement] = $dao->$key;
6100           }
6101           else {
6102             $values[$key] = $dao->$key;
6103           }
6104         }
6105       }
6106     }
6107     if (!$usedForAPI) {
6108       foreach ($this->legacyHackedFields as $realField => $labelField) {
6109         // This is a temporary routine for handling these fields while
6110         // we figure out how to handled them based on metadata in
6111         /// export and search builder. CRM-19815, CRM-19830.
6112         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6113           $dao->$realField = $dao->$labelField;
6114         }
6115       }
6116     }
6117     return $values;
6118   }
6119
6120   /**
6121    * Include pseudo fields LEFT JOIN.
6122    * @param string|array $sort can be a object or string
6123    *
6124    * @return array|NULL
6125    */
6126   public function includePseudoFieldsJoin($sort) {
6127     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6128       return NULL;
6129     }
6130     $sort = is_string($sort) ? $sort : $sort->orderBy();
6131     $present = [];
6132
6133     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6134       if (!empty($value['table'])) {
6135         $regex = "/({$value['table']}\.|{$name})/";
6136         if (preg_match($regex, $sort)) {
6137           $this->_elemnt[$value['element']] = 1;
6138           $this->_select[$value['element']] = $value['select'];
6139           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6140           $present[$value['table']] = $value['join'];
6141         }
6142       }
6143     }
6144     $presentSimpleFrom = $present;
6145
6146     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6147       array_key_exists('civicrm_country', $presentSimpleFrom)
6148     ) {
6149       unset($presentSimpleFrom['civicrm_country']);
6150     }
6151     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6152       array_key_exists('civicrm_country', $present)
6153     ) {
6154       unset($present['civicrm_country']);
6155     }
6156
6157     $presentClause = $presentSimpleFromClause = NULL;
6158     if (!empty($present)) {
6159       $presentClause = implode(' ', $present);
6160     }
6161     if (!empty($presentSimpleFrom)) {
6162       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6163     }
6164
6165     $this->_fromClause = $this->_fromClause . $presentClause;
6166     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6167
6168     return [$presentClause, $presentSimpleFromClause];
6169   }
6170
6171   /**
6172    * Build qill for field.
6173    *
6174    * Qill refers to the query detail visible on the UI.
6175    *
6176    * @param string $daoName
6177    * @param string $fieldName
6178    * @param mixed $fieldValue
6179    * @param string $op
6180    * @param array $pseudoExtraParam
6181    * @param int $type
6182    *   Type of the field per CRM_Utils_Type
6183    *
6184    * @return array
6185    */
6186   public static function buildQillForFieldValue(
6187     $daoName,
6188     $fieldName,
6189     $fieldValue,
6190     $op,
6191     $pseudoExtraParam = [],
6192     $type = CRM_Utils_Type::T_STRING
6193   ) {
6194     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6195
6196     //API usually have fieldValue format as array(operator => array(values)),
6197     //so we need to separate operator out of fieldValue param
6198     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6199       $op = key($fieldValue);
6200       $fieldValue = $fieldValue[$op];
6201     }
6202
6203     // if Operator chosen is NULL/EMPTY then
6204     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6205       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6206     }
6207
6208     // @todo - if the right BAO is passed in special handling for the below
6209     // fields should not be required. testQillOptions.
6210     if ($fieldName == 'country_id') {
6211       $pseudoOptions = CRM_Core_PseudoConstant::country();
6212     }
6213     elseif ($fieldName == 'county_id') {
6214       $pseudoOptions = CRM_Core_PseudoConstant::county();
6215     }
6216     elseif ($fieldName == 'world_region') {
6217       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6218     }
6219     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6220       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6221       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6222     }
6223     elseif ($fieldName == 'contribution_product_id') {
6224       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6225     }
6226     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6227       $pseudoOptions = CRM_Core_PseudoConstant::group();
6228     }
6229     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6230       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6231     }
6232     elseif ($daoName) {
6233       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6234     }
6235
6236     if (is_array($fieldValue)) {
6237       $qillString = [];
6238       if (!empty($pseudoOptions)) {
6239         foreach ((array) $fieldValue as $val) {
6240           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6241         }
6242         $fieldValue = implode(', ', $qillString);
6243       }
6244       else {
6245         if ($type == CRM_Utils_Type::T_DATE) {
6246           foreach ($fieldValue as $index => $value) {
6247             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6248           }
6249         }
6250         $separator = ', ';
6251         // @todo - this is a bit specific (one operator).
6252         // However it is covered by a unit test so can be altered later with
6253         // some confidence.
6254         if ($op === 'BETWEEN') {
6255           $separator = ' AND ';
6256         }
6257         $fieldValue = implode($separator, $fieldValue);
6258       }
6259     }
6260     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6261       $fieldValue = $pseudoOptions[$fieldValue];
6262     }
6263     elseif ($type === CRM_Utils_Type::T_DATE) {
6264       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6265     }
6266
6267     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6268   }
6269
6270   /**
6271    * Get the qill (search description for field) for the specified field.
6272    *
6273    * @param string $daoName
6274    * @param string $name
6275    * @param string $value
6276    * @param string|array $op
6277    * @param string $label
6278    *
6279    * @return string
6280    */
6281   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6282     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6283     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6284   }
6285
6286   /**
6287    * Alter value to reflect wildcard settings.
6288    *
6289    * The form will have tried to guess whether this is a good field to wildcard but there is
6290    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6291    * or only the end of the string
6292    *
6293    * @param bool $wildcard
6294    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6295    * @param string $op
6296    *   Generally '=' or 'LIKE'.
6297    * @param string $value
6298    *   The search string.
6299    *
6300    * @return string
6301    */
6302   public static function getWildCardedValue($wildcard, $op, $value) {
6303     if ($wildcard && $op === 'LIKE') {
6304       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6305         return "%$value%";
6306       }
6307       else {
6308         return "$value%";
6309       }
6310     }
6311     else {
6312       return "$value";
6313     }
6314   }
6315
6316   /**
6317    * Process special fields of Search Form in OK (Operator in Key) format
6318    *
6319    * @param array $formValues
6320    * @param array $specialFields
6321    *    Special params to be processed
6322    * @param array $changeNames
6323    *   Array of fields whose name should be changed
6324    */
6325   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6326     // Array of special fields whose value are considered only for NULL or EMPTY operators
6327     $nullableFields = ['contribution_batch_id'];
6328
6329     foreach ($specialFields as $element) {
6330       $value = $formValues[$element] ?? NULL;
6331       if ($value) {
6332         if (is_array($value)) {
6333           if (array_key_exists($element, $changeNames)) {
6334             unset($formValues[$element]);
6335             $element = $changeNames[$element];
6336           }
6337           $formValues[$element] = ['IN' => $value];
6338         }
6339         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6340           $formValues[$element] = [$value => 1];
6341         }
6342         elseif (!in_array($element, $nullableFields)) {
6343           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6344           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6345           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6346         }
6347       }
6348     }
6349   }
6350
6351   /**
6352    * Parse and assimilate the various sort options.
6353    *
6354    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6355    * `email`), the related table may be joined automatically.
6356    *
6357    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6358    * it's been extracted to a standalone function.
6359    *
6360    * @param string|CRM_Utils_Sort $sort
6361    *   The order by string.
6362    * @param null $sortOrder
6363    *   Who knows? Hu knows. He who knows Hu knows who.
6364    *
6365    * @return string
6366    *   list(string $orderByClause, string $additionalFromClause).
6367    *
6368    * @throws \CRM_Core_Exception
6369    */
6370   protected function prepareOrderBy($sort, $sortOrder) {
6371     $orderByArray = [];
6372     $orderBy = '';
6373
6374     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6375       isset($this->_distinctComponentClause)
6376     ) {
6377       if ($sort) {
6378         if (is_string($sort)) {
6379           $orderBy = $sort;
6380         }
6381         else {
6382           $orderBy = trim($sort->orderBy());
6383         }
6384         // Deliberately remove the backticks again, as they mess up the evil
6385         // string munging below. This balanced by re-escaping before use.
6386         $orderBy = str_replace('`', '', $orderBy);
6387
6388         if (!empty($orderBy)) {
6389           // this is special case while searching for
6390           // change log CRM-1718
6391           if (preg_match('/sort_name/i', $orderBy)) {
6392             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6393           }
6394
6395           if ($sortOrder) {
6396             $orderBy .= " $sortOrder";
6397           }
6398
6399           // always add contact_a.id to the ORDER clause
6400           // so the order is deterministic
6401           if (strpos('contact_a.id', $orderBy) === FALSE) {
6402             $orderBy .= ", contact_a.id";
6403           }
6404         }
6405       }
6406       else {
6407         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6408       }
6409     }
6410     if (!$orderBy) {
6411       return NULL;
6412     }
6413     // Remove this here & add it at the end for simplicity.
6414     $order = trim($orderBy);
6415     $orderByArray = explode(',', $order);
6416
6417     foreach ($orderByArray as $orderByClause) {
6418       $orderByClauseParts = explode(' ', trim($orderByClause));
6419       $field = $orderByClauseParts[0];
6420       $direction = $orderByClauseParts[1] ?? 'asc';
6421       $fieldSpec = $this->getMetadataForRealField($field);
6422
6423       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6424       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6425       // add keys to $this->fields to represent the actual keys for locations.
6426       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6427         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6428         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6429       }
6430
6431       if ($this->_returnProperties === []) {
6432         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6433           $this->_tables[$fieldSpec['table_name']] = 1;
6434           $order = $fieldSpec['where'] . ' ' . $direction;
6435         }
6436
6437       }
6438       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6439       // add to cfIDs array if not present
6440       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6441         $this->_cfIDs[$cfID] = [];
6442         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6443         $this->_customQuery->query();
6444         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6445         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6446       }
6447
6448       // By replacing the join to the option value table with the mysql construct
6449       // ORDER BY field('contribution_status_id', 2,1,4)
6450       // we can remove a join. In the case of the option value join it is
6451       /// a join known to cause slow queries.
6452       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6453       // matching an id+name parrern in the
6454       // first instance for scope reasons. They require slightly different handling as the column (label)
6455       // is not declared for them.
6456       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6457       // escaping.
6458       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6459       if (!empty($pseudoConstantMetadata)
6460       ) {
6461         if (!empty($pseudoConstantMetadata['optionGroupName'])
6462           || $this->isPseudoFieldAnFK($fieldSpec)
6463         ) {
6464           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6465           if (empty($fieldSpec['bao'])) {
6466             continue;
6467           }
6468           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name']);
6469           natcasesort($sortedOptions);
6470           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6471           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6472           // this might give some early warning.
6473           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6474           // use where if it's set to fully qualify ambiguous column names
6475           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6476           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6477           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6478         }
6479         //CRM-12565 add "`" around $field if it is a pseudo constant
6480         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6481         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6482           $order = str_replace($field, "`{$field}`", $order);
6483         }
6484       }
6485     }
6486
6487     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6488     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6489
6490     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6491     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6492     if ($order) {
6493       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6494       return ' ORDER BY ' . $order;
6495     }
6496   }
6497
6498   /**
6499    * Convert a string of group IDs to a string of group labels.
6500    *
6501    * The original string may include duplicates and groups the user does not have
6502    * permission to see.
6503    *
6504    * @param CRM_Core_DAO $dao
6505    * @param string $val
6506    *
6507    * @return string
6508    */
6509   public function convertGroupIDStringToLabelString(&$dao, $val) {
6510     $groupIDs = explode(',', $val);
6511     // Note that groups that the user does not have permission to will be excluded (good).
6512     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6513     return implode(', ', $groups);
6514
6515   }
6516
6517   /**
6518    * Set the qill and where properties for a field.
6519    *
6520    * This function is intended as a short-term function to encourage refactoring
6521    * & re-use - but really we should just have less special-casing.
6522    *
6523    * @param string $name
6524    * @param string $op
6525    * @param string|array $value
6526    * @param string $grouping
6527    * @param array $field
6528    *
6529    * @throws \CRM_Core_Exception
6530    */
6531   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6532     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6533     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6534     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6535       1 => $field['title'],
6536       2 => $qillop,
6537       3 => $qillVal,
6538     ]);
6539   }
6540
6541   /**
6542    * Has the pseudoconstant of the field been requested.
6543    *
6544    * For example if the field is payment_instrument_id then it
6545    * has been requested if either payment_instrument_id or payment_instrument
6546    * have been requested. Payment_instrument is the option groun name field value.
6547    *
6548    * @param array $field
6549    * @param string $fieldName
6550    *   The unique name of the field - ie. the one it will be aliased to in the query.
6551    *
6552    * @return bool
6553    */
6554   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6555     $realField = $this->getMetadataForRealField($fieldName);
6556     if (!isset($realField['pseudoconstant'])) {
6557       return FALSE;
6558     }
6559     $pseudoConstant = $realField['pseudoconstant'];
6560     if (empty($pseudoConstant['optionGroupName']) &&
6561       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6562       // We are increasing our pseudoconstant handling - but still very cautiously,
6563       // hence the check for labelColumn === name
6564       return FALSE;
6565     }
6566
6567     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6568       return TRUE;
6569     }
6570     if (!empty($this->_returnProperties[$fieldName])) {
6571       return TRUE;
6572     }
6573     // Is this still required - the above goes off the unique name. Test with things like
6574     // communication_preferences & prefix_id.
6575     if (!empty($this->_returnProperties[$field['name']])) {
6576       return TRUE;
6577     }
6578     return FALSE;
6579   }
6580
6581   /**
6582    * Get Select Clause.
6583    *
6584    * @return string
6585    */
6586   public function getSelect() {
6587     $select = 'SELECT ';
6588     if (isset($this->_distinctComponentClause)) {
6589       $select .= "{$this->_distinctComponentClause}, ";
6590     }
6591     $select .= implode(', ', $this->_select);
6592     return $select;
6593   }
6594
6595   /**
6596    * Add basic statistics to the summary.
6597    *
6598    * @param array $summary
6599    * @param string $where
6600    * @param string $from
6601    *
6602    * @return array
6603    * @throws \CRM_Core_Exception
6604    */
6605   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6606     $summary['total']['count'] = 0;
6607     $summary['total']['amount'] = $summary['total']['avg'] = [];
6608
6609     $query = "
6610       SELECT COUNT( conts.total_amount ) as total_count,
6611         SUM(   conts.total_amount ) as total_amount,
6612         AVG(   conts.total_amount ) as total_avg,
6613         conts.currency              as currency
6614       FROM (
6615         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6616         civicrm_contribution.currency
6617         $from
6618         $where AND civicrm_contribution.contribution_status_id = 1
6619       GROUP BY civicrm_contribution.id
6620     ) as conts
6621     GROUP BY currency";
6622
6623     $dao = CRM_Core_DAO::executeQuery($query);
6624
6625     while ($dao->fetch()) {
6626       $summary['total']['count'] += $dao->total_count;
6627       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6628       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6629     }
6630
6631     if (!empty($summary['total']['amount'])) {
6632       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6633       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6634     }
6635     else {
6636       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6637     }
6638     return $summary;
6639   }
6640
6641   /**
6642    * Add basic soft credit statistics to summary array.
6643    *
6644    * @param array $summary
6645    * @param string $where
6646    * @param string $from
6647    *
6648    * @throws \CRM_Core_Exception
6649    */
6650   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6651     $query = "
6652       SELECT COUNT( conts.total_amount ) as total_count,
6653       SUM(   conts.total_amount ) as total_amount,
6654       AVG(   conts.total_amount ) as total_avg,
6655       conts.currency as currency
6656       FROM (
6657         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6658         $from
6659         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6660         GROUP BY civicrm_contribution_soft.id
6661       ) as conts
6662       GROUP BY currency";
6663
6664     $dao = CRM_Core_DAO::executeQuery($query);
6665     $summary['soft_credit']['count'] = 0;
6666     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6667     while ($dao->fetch()) {
6668       $summary['soft_credit']['count'] += $dao->total_count;
6669       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6670       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6671     }
6672     if (!empty($summary['soft_credit']['amount'])) {
6673       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6674       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6675     }
6676     else {
6677       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6678     }
6679   }
6680
6681   /**
6682    * Add basic stats about cancelled contributions to the summary.
6683    *
6684    * @param array $summary
6685    * @param string $where
6686    * @param string $from
6687    *
6688    * @throws \CRM_Core_Exception
6689    */
6690   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6691     $query = "
6692       SELECT COUNT( conts.total_amount ) as cancel_count,
6693        SUM(   conts.total_amount ) as cancel_amount,
6694        AVG(   conts.total_amount ) as cancel_avg,
6695        conts.currency              as currency
6696         FROM (
6697       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6698       $from
6699       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6700       GROUP BY civicrm_contribution.id
6701     ) as conts
6702     GROUP BY currency";
6703
6704     $dao = CRM_Core_DAO::executeQuery($query);
6705
6706     if ($dao->N <= 1) {
6707       if ($dao->fetch()) {
6708         $summary['cancel']['count'] = $dao->cancel_count;
6709         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6710         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6711       }
6712     }
6713     else {
6714       $summary['cancel']['count'] = 0;
6715       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6716       while ($dao->fetch()) {
6717         $summary['cancel']['count'] += $dao->cancel_count;
6718         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6719         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6720       }
6721       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6722       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6723     }
6724   }
6725
6726   /**
6727    * Create the sql query for an contact search.
6728    *
6729    * @param int $offset
6730    *   The offset for the query.
6731    * @param int $rowCount
6732    *   The number of rows to return.
6733    * @param string|CRM_Utils_Sort $sort
6734    *   The order by string.
6735    * @param bool $count
6736    *   Is this a count only query ?.
6737    * @param bool $includeContactIds
6738    *   Should we include contact ids?.
6739    * @param bool $sortByChar
6740    *   If true returns the distinct array of first characters for search results.
6741    * @param bool $groupContacts
6742    *   If true, return only the contact ids.
6743    * @param string $additionalWhereClause
6744    *   If the caller wants to further restrict the search (used for components).
6745    * @param null $sortOrder
6746    * @param string $additionalFromClause
6747    *   Should be clause with proper joins, effective to reduce where clause load.
6748    *
6749    * @param bool $skipOrderAndLimit
6750    *
6751    * @return string
6752    *
6753    * @throws \CRM_Core_Exception
6754    */
6755   public function getSearchSQL(
6756     $offset = 0, $rowCount = 0, $sort = NULL,
6757     $count = FALSE, $includeContactIds = FALSE,
6758     $sortByChar = FALSE, $groupContacts = FALSE,
6759     $additionalWhereClause = NULL, $sortOrder = NULL,
6760     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6761
6762     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6763
6764     if ($sortByChar) {
6765       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6766       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6767     }
6768
6769     if ($skipOrderAndLimit) {
6770       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6771       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6772     }
6773     else {
6774       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6775     }
6776     return $query;
6777   }
6778
6779   /**
6780    * Get the component parts of the search query as an array.
6781    *
6782    * @param int $offset
6783    *   The offset for the query.
6784    * @param int $rowCount
6785    *   The number of rows to return.
6786    * @param string|CRM_Utils_Sort $sort
6787    *   The order by string.
6788    * @param bool $count
6789    *   Is this a count only query ?.
6790    * @param bool $includeContactIds
6791    *   Should we include contact ids?.
6792    * @param bool $sortByChar
6793    *   If true returns the distinct array of first characters for search results.
6794    * @param bool $groupContacts
6795    *   If true, return only the contact ids.
6796    * @param string $additionalWhereClause
6797    *   If the caller wants to further restrict the search (used for components).
6798    * @param null $sortOrder
6799    * @param string $additionalFromClause
6800    *   Should be clause with proper joins, effective to reduce where clause load.
6801    *
6802    * @return array
6803    * @throws \CRM_Core_Exception
6804    */
6805   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6806     $count = FALSE, $includeContactIds = FALSE,
6807     $sortByChar = FALSE, $groupContacts = FALSE,
6808     $additionalWhereClause = NULL, $sortOrder = NULL,
6809     $additionalFromClause = NULL) {
6810     if ($includeContactIds) {
6811       $this->_includeContactIds = TRUE;
6812       $this->_whereClause = $this->whereClause();
6813     }
6814     $onlyDeleted = in_array([
6815       'deleted_contacts',
6816       '=',
6817       '1',
6818       '0',
6819       '0',
6820     ], $this->_params);
6821
6822     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6823     // and that contact happens to be deleted, set $onlyDeleted to true
6824     foreach ($this->_params as $values) {
6825       $name = $values[0] ?? NULL;
6826       $op = $values[1] ?? NULL;
6827       $value = $values[2] ?? NULL;
6828       if ($name === 'contact_id' and $op === '=') {
6829         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6830           $onlyDeleted = TRUE;
6831         }
6832         break;
6833       }
6834     }
6835
6836     // building the query string
6837     $groupBy = $groupByCols = NULL;
6838     if (!$count) {
6839       if (isset($this->_groupByComponentClause)) {
6840         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6841         $groupByCols = explode(', ', $groupByCols);
6842       }
6843       elseif ($this->_useGroupBy) {
6844         $groupByCols = ['contact_a.id'];
6845       }
6846     }
6847     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6848       $groupByCols = ['civicrm_activity.id'];
6849     }
6850     if (!empty($groupByCols)) {
6851       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6852     }
6853
6854     $order = $orderBy = '';
6855     if (!$count) {
6856       if (!$sortByChar) {
6857         $order = $this->prepareOrderBy($sort, $sortOrder);
6858       }
6859     }
6860     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6861     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6862     //     clause. This will impact the search query output.
6863     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6864
6865     if ($disableFullGroupByMode) {
6866       CRM_Core_DAO::disableFullGroupByMode();
6867     }
6868
6869     // CRM-15231
6870     $this->_sort = $sort;
6871
6872     //CRM-15967
6873     $this->includePseudoFieldsJoin($sort);
6874
6875     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6876
6877     if ($additionalWhereClause) {
6878       $where = $where . ' AND ' . $additionalWhereClause;
6879     }
6880
6881     //additional from clause should be w/ proper joins.
6882     if ($additionalFromClause) {
6883       $from .= "\n" . $additionalFromClause;
6884     }
6885
6886     // if we are doing a transform, do it here
6887     // use the $from, $where and $having to get the contact ID
6888     if ($this->_displayRelationshipType) {
6889       $this->filterRelatedContacts($from, $where, $having);
6890     }
6891     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6892
6893     return [
6894       'select' => $select,
6895       'from' => $from,
6896       'where' => $where,
6897       'order_by' => $order,
6898       'group_by' => $groupBy,
6899       'having' => $having,
6900       'limit' => $limit,
6901     ];
6902   }
6903
6904   /**
6905    * Get the metadata for a given field.
6906    *
6907    * @param string $fieldName
6908    *
6909    * @return array
6910    */
6911   protected function getMetadataForField($fieldName) {
6912     if ($fieldName === 'contact_a.id') {
6913       // This seems to be the only anomaly.
6914       $fieldName = 'id';
6915     }
6916     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6917     $field = $this->_fields[$fieldName] ?? $pseudoField;
6918     $field = array_merge($field, $pseudoField);
6919     if (!empty($field) && empty($field['name'])) {
6920       // standardising field formatting here - over time we can phase out variants.
6921       // all paths using this currently unit tested
6922       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6923     }
6924     return $field;
6925   }
6926
6927   /**
6928    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6929    *
6930    * @param string $fieldName
6931    *
6932    * @return array
6933    */
6934   public function getMetadataForRealField($fieldName) {
6935     $field = $this->getMetadataForField($fieldName);
6936     if (!empty($field['is_pseudofield_for'])) {
6937       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6938       $field['pseudofield_name'] = $fieldName;
6939     }
6940     elseif (!empty($field['pseudoconstant'])) {
6941       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6942         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6943         if (empty($field['table_name'])) {
6944           if (!empty($field['where'])) {
6945             $field['table_name'] = explode('.', $field['where'])[0];
6946           }
6947           else {
6948             $field['table_name'] = 'civicrm_contact';
6949           }
6950         }
6951       }
6952     }
6953     return $field;
6954   }
6955
6956   /**
6957    * Get the field datatype, using the type in the database rather than the pseudofield, if a pseudofield.
6958    *
6959    * @param string $fieldName
6960    *
6961    * @return string
6962    */
6963   public function getDataTypeForRealField($fieldName) {
6964     return CRM_Utils_Type::typeToString($this->getMetadataForRealField($fieldName)['type']);
6965   }
6966
6967   /**
6968    * If we have a field that is better rendered via the pseudoconstant handled them here.
6969    *
6970    * Rather than joining in the additional table we render the option value on output.
6971    *
6972    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6973    * carefully with test coverage but aim to extend.
6974    *
6975    * @param string $name
6976    */
6977   protected function addPseudoconstantFieldToSelect($name) {
6978     $field = $this->getMetadataForRealField($name);
6979     $realFieldName = $field['name'];
6980     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6981     if ($pseudoFieldName) {
6982       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6983       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6984         'pseudoField' => $pseudoFieldName,
6985         'idCol' => $realFieldName,
6986         'field_name' => $field['name'],
6987         'bao' => $field['bao'],
6988         'pseudoconstant' => $field['pseudoconstant'],
6989       ];
6990     }
6991
6992     $this->_tables[$field['table_name']] = 1;
6993     $this->_element[$realFieldName] = 1;
6994     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6995   }
6996
6997   /**
6998    * Is this pseudofield a foreign key constraint.
6999    *
7000    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
7001    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
7002    * array which is done for contributions which are mostly handled as pseudoconstants.
7003    *
7004    * @param $fieldSpec
7005    *
7006    * @return bool
7007    */
7008   protected function isPseudoFieldAnFK($fieldSpec) {
7009     if (empty($fieldSpec['FKClassName'])
7010       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
7011       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
7012       return FALSE;
7013     }
7014     return TRUE;
7015   }
7016
7017   /**
7018    * Is the field a relative date field.
7019    *
7020    * @param string $fieldName
7021    *
7022    * @return bool
7023    */
7024   protected function isARelativeDateField($fieldName) {
7025     if (substr($fieldName, -9, 9) !== '_relative') {
7026       return FALSE;
7027     }
7028     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
7029     return isset($this->_fields[$realField]);
7030   }
7031
7032   /**
7033    * Get the specifications for the field, if available.
7034    *
7035    * @param string $fieldName
7036    *   Fieldname as displayed on the form.
7037    *
7038    * @return array
7039    */
7040   public function getFieldSpec($fieldName) {
7041     if (isset($this->_fields[$fieldName])) {
7042       $fieldSpec = $this->_fields[$fieldName];
7043       if (!empty($fieldSpec['is_pseudofield_for'])) {
7044         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
7045       }
7046       return $fieldSpec;
7047     }
7048     $lowFieldName = str_replace('_low', '', $fieldName);
7049     if (isset($this->_fields[$lowFieldName])) {
7050       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
7051     }
7052     $highFieldName = str_replace('_high', '', $fieldName);
7053     if (isset($this->_fields[$highFieldName])) {
7054       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
7055     }
7056     return [];
7057   }
7058
7059   public function buildWhereForDate() {
7060
7061   }
7062
7063   /**
7064    * Is the field a relative date field.
7065    *
7066    * @param string $fieldName
7067    *
7068    * @return bool
7069    */
7070   protected function isADateRangeField($fieldName) {
7071     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7072       return FALSE;
7073     }
7074     return !empty($this->getFieldSpec($fieldName));
7075   }
7076
7077   /**
7078    * @param $values
7079    */
7080   protected function buildRelativeDateQuery(&$values) {
7081     $value = $values[2] ?? NULL;
7082     if (empty($value)) {
7083       return;
7084     }
7085     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7086     $fieldSpec = $this->_fields[$fieldName];
7087     $tableName = $fieldSpec['table_name'];
7088     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7089     $grouping = $values[3] ?? NULL;
7090     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7091     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7092     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7093
7094     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7095     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7096     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7097
7098     $where = $fieldSpec['where'];
7099     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7100       // Special handling for contact table as it has a known alias in advanced search.
7101       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7102       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7103     }
7104
7105     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7106     if ($fieldName === 'relation_active_period_date') {
7107       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7108       // Not sure the  'right' fix.
7109       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7110       return;
7111     }
7112
7113     if (empty($dates[0])) {
7114       // ie. no start date we only have end date
7115       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7116     }
7117     elseif (empty($dates[1])) {
7118
7119       // ie. no end date we only have start date
7120       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7121     }
7122     else {
7123       // we have start and end dates.
7124       if ($secondWhere !== $where) {
7125         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7126       }
7127       else {
7128         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7129       }
7130     }
7131   }
7132
7133   /**
7134    * Build the query for a date field if it is a _high or _low field.
7135    *
7136    * @param $values
7137    *
7138    * @return bool
7139    * @throws \CRM_Core_Exception
7140    */
7141   public function buildDateRangeQuery($values) {
7142     if ($this->isADateRangeField($values[0])) {
7143       $fieldSpec = $this->getFieldSpec($values[0]);
7144       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7145       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7146       return TRUE;
7147     }
7148     return FALSE;
7149   }
7150
7151   /**
7152    * Add the address table into the query.
7153    *
7154    * @param string $tableKey
7155    * @param string $joinCondition
7156    *
7157    * @return array
7158    *   - alias name
7159    *   - address join.
7160    */
7161   protected function addAddressTable($tableKey, $joinCondition) {
7162     $tName = "$tableKey-address";
7163     $aName = "`$tableKey-address`";
7164     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7165     $this->_element["{$tName}_id"] = 1;
7166     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7167     $this->_tables[$tName] = $addressJoin;
7168
7169     return [
7170       $aName,
7171       $addressJoin,
7172     ];
7173   }
7174
7175   /**
7176    * Get the clause for group status.
7177    *
7178    * @param int $grouping
7179    *
7180    * @return string
7181    */
7182   protected function getGroupStatusClause($grouping) {
7183     $statuses = $this->getSelectedGroupStatuses($grouping);
7184     return "status IN (" . implode(', ', $statuses) . ")";
7185   }
7186
7187   /**
7188    * Get an array of the statuses that have been selected.
7189    *
7190    * @param string $grouping
7191    *
7192    * @return array
7193    *
7194    * @throws \CRM_Core_Exception
7195    */
7196   protected function getSelectedGroupStatuses($grouping) {
7197     $statuses = [];
7198     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7199     if ($gcsValues &&
7200       is_array($gcsValues[2])
7201     ) {
7202       foreach ($gcsValues[2] as $k => $v) {
7203         if ($v) {
7204           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7205         }
7206       }
7207     }
7208     else {
7209       $statuses[] = "'Added'";
7210     }
7211     return $statuses;
7212   }
7213
7214   /**
7215    * Get the qill value for the field.
7216    *
7217    * @param string $name
7218    * @param array|int|string $value
7219    * @param string $op
7220    * @param array $fieldSpec
7221    * @param string $labelOverride
7222    *   Label override, if required.
7223    *
7224    * @return string
7225    */
7226   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7227     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7228     return (string) ts("%1 %2 %3", [
7229       1 => $labelOverride ?? $fieldSpec['title'],
7230       2 => $qillop,
7231       3 => $qillVal,
7232     ]);
7233   }
7234
7235   /**
7236    * Where handling for any field with adequately defined metadata.
7237    *
7238    * @param array $fieldSpec
7239    * @param string $name
7240    * @param string|array|int $value
7241    * @param string $op
7242    * @param string|int $grouping
7243    *
7244    * @throws \CRM_Core_Exception
7245    */
7246   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7247     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7248     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7249     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7250       $this->_tables[$fieldSpec['table_name']] = 1;
7251     }
7252     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7253       $this->_whereTables[$fieldSpec['table_name']] = 1;
7254     }
7255   }
7256
7257   /**
7258    * Get the qill for the relative date range.
7259    *
7260    * @param string|null $from
7261    * @param string|null $to
7262    * @param string $fieldTitle
7263    * @param string $relativeRange
7264    *
7265    * @return string
7266    */
7267   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7268     if (!$from) {
7269       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7270     }
7271     if (!$to) {
7272       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7273     }
7274     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7275       CRM_Utils_Date::customFormat($from),
7276       CRM_Utils_Date::customFormat($to),
7277     ]) . ')';
7278   }
7279
7280 }