CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * This is the heart of the search query building mechanism.
  20  */
  21 class CRM_Contact_BAO_Query {
  22
  23   /**
  24    * The various search modes.
  25    *
  26    * As of February 2017, entries not present for 4, 32, 64, 1024.
  27    *
  28    * MODE_ALL seems to be out of sync with the available constants;
  29    * if this is intentionally excluding MODE_MAILING then that may
  30    * bear documenting?
  31    *
  32    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  33    *
  34    * @var int
  35    */
  36   const
  37     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  38     MODE_CONTACTS = 1,
  39     MODE_CONTRIBUTE = 2,
  40     // There is no 4,
  41     MODE_MEMBER = 8,
  42     MODE_EVENT = 16,
  43     MODE_CONTACTSRELATED = 32,
  44     // no 64.
  45     MODE_GRANT = 128,
  46     MODE_PLEDGEBANK = 256,
  47     MODE_PLEDGE = 512,
  48     // There is no 1024,
  49     MODE_CASE = 2048,
  50     MODE_ACTIVITY = 4096,
  51     MODE_CAMPAIGN = 8192,
  52     MODE_MAILING = 16384,
  53     MODE_ALL = 17407;
  54
  55   /**
  56    * Constants for search operators
  57    */
  58   const
  59     SEARCH_OPERATOR_AND = 'AND',
  60     SEARCH_OPERATOR_OR = 'OR';
  61
  62   /**
  63    * The default set of return properties.
  64    *
  65    * @var array
  66    */
  67   public static $_defaultReturnProperties;
  68
  69   /**
  70    * The default set of hier return properties.
  71    *
  72    * @var array
  73    */
  74   public static $_defaultHierReturnProperties;
  75
  76   /**
  77    * The set of input params.
  78    *
  79    * @var array
  80    */
  81   public $_params;
  82
  83   public $_cfIDs;
  84
  85   public $_paramLookup;
  86
  87   public $_sort;
  88
  89   /**
  90    * The set of output params
  91    *
  92    * @var array
  93    */
  94   public $_returnProperties;
  95
  96   /**
  97    * The select clause
  98    *
  99    * @var array
 100    */
 101   public $_select;
 102
 103   /**
 104    * The name of the elements that are in the select clause
 105    * used to extract the values.
 106    *
 107    * @var array
 108    */
 109   public $_element;
 110
 111   /**
 112    * The tables involved in the query.
 113    *
 114    * @var array
 115    */
 116   public $_tables;
 117
 118   /**
 119    * The table involved in the where clause.
 120    *
 121    * @var array
 122    */
 123   public $_whereTables;
 124
 125   /**
 126    * Array of WHERE clause components.
 127    *
 128    * @var array
 129    */
 130   public $_where;
 131
 132   /**
 133    * The WHERE clause as a string.
 134    *
 135    * @var string
 136    */
 137   public $_whereClause;
 138
 139   /**
 140    * Additional WHERE clause for permissions.
 141    *
 142    * @var string
 143    */
 144   public $_permissionWhereClause;
 145
 146   /**
 147    * The from string
 148    *
 149    * @var string
 150    */
 151   public $_fromClause;
 152
 153   /**
 154    * Additional permission from clause
 155    *
 156    * @var string
 157    */
 158   public $_permissionFromClause;
 159
 160   /**
 161    * The from clause for the simple select and alphabetical
 162    * select
 163    *
 164    * @var string
 165    */
 166   public $_simpleFromClause;
 167
 168   /**
 169    * The having values
 170    *
 171    * @var array
 172    */
 173   public $_having;
 174
 175   /**
 176    * The english language version of the query
 177    *
 178    * @var array
 179    */
 180   public $_qill;
 181
 182   /**
 183    * All the fields that could potentially be involved in
 184    * this query
 185    *
 186    * @var array
 187    */
 188   public $_fields;
 189
 190   /**
 191    * Fields hacked for legacy reasons.
 192    *
 193    * Generally where a field has a option group defining it's options we add them to
 194    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 195    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 196    *
 197    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 198    * field. The heading for the pseudofield would come form the the option group name & for the id field
 199    * from the xml.
 200    *
 201    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 202    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 203    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 204    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 205    * not fully agreed here.
 206    *
 207    * @var array
 208    */
 209   protected $legacyHackedFields = [
 210     'gender_id' => 'gender',
 211     'prefix_id' => 'individual_prefix',
 212     'suffix_id' => 'individual_suffix',
 213     'communication_style_id' => 'communication_style',
 214   ];
 215
 216   /**
 217    * Are we in search mode.
 218    *
 219    * @var bool
 220    */
 221   public $_search = TRUE;
 222
 223   /**
 224    * Should we skip permission checking.
 225    *
 226    * @var bool
 227    */
 228   public $_skipPermission = FALSE;
 229
 230   /**
 231    * Should we skip adding of delete clause.
 232    *
 233    * @var bool
 234    */
 235   public $_skipDeleteClause = FALSE;
 236
 237   /**
 238    * Are we in strict mode (use equality over LIKE)
 239    *
 240    * @var bool
 241    */
 242   public $_strict = FALSE;
 243
 244   /**
 245    * What operator to use to group the clauses.
 246    *
 247    * @var string
 248    */
 249   public $_operator = 'AND';
 250
 251   public $_mode = 1;
 252
 253   /**
 254    * Should we only search on primary location.
 255    *
 256    * @var bool
 257    */
 258   public $_primaryLocation = TRUE;
 259
 260   /**
 261    * Are contact ids part of the query.
 262    *
 263    * @var bool
 264    */
 265   public $_includeContactIds = FALSE;
 266
 267   /**
 268    * Should we use the smart group cache.
 269    *
 270    * @var bool
 271    */
 272   public $_smartGroupCache = TRUE;
 273
 274   /**
 275    * Should we display contacts with a specific relationship type.
 276    *
 277    * @var string
 278    */
 279   public $_displayRelationshipType;
 280
 281   /**
 282    * Reference to the query object for custom values.
 283    *
 284    * @var Object
 285    */
 286   public $_customQuery;
 287
 288   /**
 289    * Should we enable the distinct clause, used if we are including
 290    * more than one group
 291    *
 292    * @var bool
 293    */
 294   public $_useDistinct = FALSE;
 295
 296   /**
 297    * Should we just display one contact record
 298    * @var bool
 299    */
 300   public $_useGroupBy = FALSE;
 301
 302   /**
 303    * The relationship type direction
 304    *
 305    * @var array
 306    */
 307   public static $_relType;
 308
 309   /**
 310    * The activity role
 311    *
 312    * @var array
 313    */
 314   public static $_activityRole;
 315
 316   /**
 317    * Consider the component activity type
 318    * during activity search.
 319    *
 320    * @var array
 321    */
 322   public static $_considerCompActivities;
 323
 324   /**
 325    * Consider with contact activities only,
 326    * during activity search.
 327    *
 328    * @var array
 329    */
 330   public static $_withContactActivitiesOnly;
 331
 332   /**
 333    * Use distinct component clause for component searches
 334    *
 335    * @var string
 336    */
 337   public $_distinctComponentClause;
 338
 339   public $_rowCountClause;
 340
 341   /**
 342    * Use groupBy component clause for component searches
 343    *
 344    * @var string
 345    */
 346   public $_groupByComponentClause;
 347
 348   /**
 349    * Track open panes, useful in advance search
 350    *
 351    * @var array
 352    */
 353   public static $_openedPanes = [];
 354
 355   /**
 356    * For search builder - which custom fields are location-dependent
 357    * @var array
 358    */
 359   public $_locationSpecificCustomFields = [];
 360
 361   /**
 362    * The tables which have a dependency on location and/or address
 363    *
 364    * @var array
 365    */
 366   public static $_dependencies = [
 367     'civicrm_state_province' => 1,
 368     'civicrm_country' => 1,
 369     'civicrm_county' => 1,
 370     'civicrm_address' => 1,
 371     'civicrm_location_type' => 1,
 372   ];
 373
 374   /**
 375    * List of location specific fields.
 376    * @var array
 377    */
 378   public static $_locationSpecificFields = [
 379     'street_address',
 380     'street_number',
 381     'street_name',
 382     'street_unit',
 383     'supplemental_address_1',
 384     'supplemental_address_2',
 385     'supplemental_address_3',
 386     'city',
 387     'postal_code',
 388     'postal_code_suffix',
 389     'geo_code_1',
 390     'geo_code_2',
 391     'state_province',
 392     'country',
 393     'county',
 394     'phone',
 395     'email',
 396     'im',
 397     'address_name',
 398     'master_id',
 399     'location_type',
 400   ];
 401
 402   /**
 403    * Remember if we handle either end of a number or date range
 404    * so we can skip the other
 405    * @var array
 406    */
 407   protected $_rangeCache = [];
 408   /**
 409    * Set to true when $this->relationship is run to avoid adding twice.
 410    *
 411    * @var bool
 412    */
 413   protected $_relationshipValuesAdded = FALSE;
 414
 415   /**
 416    * Set to the name of the temp table if one has been created.
 417    *
 418    * @var string
 419    */
 420   public static $_relationshipTempTable;
 421
 422   public $_pseudoConstantsSelect = [];
 423
 424   public $_groupUniqueKey;
 425   public $_groupKeys = [];
 426
 427   /**
 428    * Class constructor which also does all the work.
 429    *
 430    * @param array $params
 431    * @param array $returnProperties
 432    * @param array $fields
 433    * @param bool $includeContactIds
 434    * @param bool $strict
 435    * @param bool|int $mode - mode the search is operating on
 436    *
 437    * @param bool $skipPermission
 438    * @param bool $searchDescendentGroups
 439    * @param bool $smartGroupCache
 440    * @param null $displayRelationshipType
 441    * @param string $operator
 442    * @param string $apiEntity
 443    * @param bool|null $primaryLocationOnly
 444    *
 445    * @throws \CRM_Core_Exception
 446    */
 447   public function __construct(
 448     $params = NULL, $returnProperties = NULL, $fields = NULL,
 449     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 450     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 451     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 452     $operator = 'AND',
 453     $apiEntity = NULL,
 454     $primaryLocationOnly = NULL
 455   ) {
 456     if ($primaryLocationOnly === NULL) {
 457       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 458     }
 459     $this->_primaryLocation = $primaryLocationOnly;
 460     $this->_params = &$params;
 461     if ($this->_params == NULL) {
 462       $this->_params = [];
 463     }
 464
 465     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 466       $this->_returnProperties = [];
 467     }
 468     elseif (empty($returnProperties)) {
 469       $this->_returnProperties = self::defaultReturnProperties($mode);
 470     }
 471     else {
 472       $this->_returnProperties = &$returnProperties;
 473     }
 474
 475     $this->_includeContactIds = $includeContactIds;
 476     $this->_strict = $strict;
 477     $this->_mode = $mode;
 478     $this->_skipPermission = $skipPermission;
 479     $this->_smartGroupCache = $smartGroupCache;
 480     $this->_displayRelationshipType = $displayRelationshipType;
 481     $this->setOperator($operator);
 482
 483     if ($fields) {
 484       $this->_fields = &$fields;
 485       $this->_search = FALSE;
 486       $this->_skipPermission = TRUE;
 487     }
 488     else {
 489       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 490       //  The legacy hacked fields will output as a string rather than their underlying type.
 491       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 492         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 493       }
 494       $relationMetadata = CRM_Contact_BAO_Relationship::fields();
 495       $relationFields = array_intersect_key($relationMetadata, array_fill_keys(['relationship_start_date', 'relationship_end_date'], 1));
 496       // No good option other than hard-coding metadata for this 'special' field in.
 497       $relationFields['relation_active_period_date'] = [
 498         'name' => 'relation_active_period_date',
 499         'type' => CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME,
 500         'title' => ts('Active Period'),
 501         'table_name' => 'civicrm_relationship',
 502         'where' => 'civicrm_relationship.start_date',
 503         'where_end' => 'civicrm_relationship.end_date',
 504         'html' => ['type' => 'SelectDate', 'formatType' => 'activityDateTime'],
 505       ];
 506       $this->_fields = array_merge($relationFields, $this->_fields);
 507
 508       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 509       unset($fields['note']);
 510       $this->_fields = array_merge($this->_fields, $fields);
 511
 512       // add activity fields
 513       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields());
 514       $this->_fields = array_merge($this->_fields, CRM_Activity_BAO_Activity::exportableFields('Case'));
 515       // Add hack as no unique name is defined for the field but the search form is in denial.
 516       $this->_fields['activity_priority_id'] = $this->_fields['priority_id'];
 517
 518       // add any fields provided by hook implementers
 519       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 520       $this->_fields = array_merge($this->_fields, $extFields);
 521     }
 522
 523     // basically do all the work once, and then reuse it
 524     $this->initialize($apiEntity);
 525   }
 526
 527   /**
 528    * Function which actually does all the work for the constructor.
 529    *
 530    * @param string $apiEntity
 531    *   The api entity being called.
 532    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 533    *
 534    * @throws \CRM_Core_Exception
 535    */
 536   public function initialize($apiEntity = NULL) {
 537     $this->_select = [];
 538     $this->_element = [];
 539     $this->_tables = [];
 540     $this->_whereTables = [];
 541     $this->_where = [];
 542     $this->_qill = [];
 543     $this->_cfIDs = [];
 544     $this->_paramLookup = [];
 545     $this->_having = [];
 546
 547     $this->_customQuery = NULL;
 548
 549     // reset cached static variables - CRM-5803
 550     self::$_activityRole = NULL;
 551     self::$_considerCompActivities = NULL;
 552     self::$_withContactActivitiesOnly = NULL;
 553
 554     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 555     $this->_element['contact_id'] = 1;
 556     $this->_tables['civicrm_contact'] = 1;
 557
 558     if (!empty($this->_params)) {
 559       $this->buildParamsLookup();
 560     }
 561
 562     $this->_whereTables = $this->_tables;
 563
 564     $this->selectClause($apiEntity);
 565     if (!empty($this->_cfIDs)) {
 566       // @todo This function is the select function but instead of running 'select' it
 567       // is running the whole query.
 568       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
 569       $this->_customQuery->query();
 570       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
 571       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
 572       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
 573     }
 574     $isForcePrimaryOnly = !empty($apiEntity);
 575     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 576     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 577       $component = 'contribution';
 578     }
 579     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 580       $component = 'membership';
 581     }
 582     if (isset($component) && !$this->_skipPermission) {
 583       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 584       $clauses = CRM_Financial_BAO_FinancialType::buildPermissionedClause($component);
 585       if (!empty($this->_whereClause) && !empty($clauses)) {
 586         $this->_whereClause .= ' AND ';
 587       }
 588       $this->_whereClause .= $clauses;
 589     }
 590
 591     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 592     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 593
 594     $this->openedSearchPanes(TRUE);
 595   }
 596
 597   /**
 598    * Function for same purpose as convertFormValues.
 599    *
 600    * Like convert form values this function exists to pre-Process parameters from the form.
 601    *
 602    * It is unclear why they are different functions & likely relates to advances search
 603    * versus search builder.
 604    *
 605    * The direction we are going is having the form convert values to a standardised format &
 606    * moving away from weird & wonderful where clause switches.
 607    *
 608    * Fix and handle contact deletion nicely.
 609    *
 610    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 611    *
 612    * @see https://issues.civicrm.org/jira/browse/CRM-11971
 613    */
 614   public function buildParamsLookup() {
 615     $trashParamExists = FALSE;
 616     $paramByGroup = [];
 617     foreach ($this->_params as $k => $param) {
 618       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 619         $trashParamExists = TRUE;
 620       }
 621       if (!empty($param[3])) {
 622         $paramByGroup[$param[3]][$k] = $param;
 623       }
 624     }
 625
 626     if ($trashParamExists) {
 627       $this->_skipDeleteClause = TRUE;
 628
 629       //cycle through group sets and explicitly add trash param if not set
 630       foreach ($paramByGroup as $setID => $set) {
 631         if (
 632           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 633           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 634         ) {
 635           $this->_params[] = [
 636             'contact_is_deleted',
 637             '=',
 638             '0',
 639             $setID,
 640             '0',
 641           ];
 642         }
 643       }
 644     }
 645
 646     foreach ($this->_params as $value) {
 647       if (empty($value[0])) {
 648         continue;
 649       }
 650       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace(['_relative', '_low', '_high', '_to', '_high'], '', $value[0]));
 651       if ($cfID) {
 652         if (!array_key_exists($cfID, $this->_cfIDs)) {
 653           $this->_cfIDs[$cfID] = [];
 654         }
 655         // Set wildcard value based on "and/or" selection
 656         foreach ($this->_params as $key => $param) {
 657           if ($param[0] == $value[0] . '_operator') {
 658             $value[4] = $param[2] == 'or';
 659             break;
 660           }
 661         }
 662         $this->_cfIDs[$cfID][] = $value;
 663       }
 664
 665       if (!array_key_exists($value[0], $this->_paramLookup)) {
 666         $this->_paramLookup[$value[0]] = [];
 667       }
 668       if ($value[0] !== 'group') {
 669         // Just trying to unravel how group interacts here! This whole function is weird.
 670         $this->_paramLookup[$value[0]][] = $value;
 671       }
 672     }
 673   }
 674
 675   /**
 676    * Some composite fields do not appear in the fields array hack to make them part of the query.
 677    *
 678    * @param $apiEntity
 679    *   The api entity being called.
 680    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 681    */
 682   public function addSpecialFields($apiEntity) {
 683     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 684     // if get called via Contact.get API having address_id as return parameter
 685     if ($apiEntity === 'Contact') {
 686       $special[] = 'address_id';
 687     }
 688     foreach ($special as $name) {
 689       if (!empty($this->_returnProperties[$name])) {
 690         if ($name === 'address_id') {
 691           $this->_tables['civicrm_address'] = 1;
 692           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 693           $this->_element['address_id'] = 1;
 694         }
 695         else {
 696           $this->_select[$name] = "contact_a.{$name} as $name";
 697           $this->_element[$name] = 1;
 698         }
 699       }
 700     }
 701   }
 702
 703   /**
 704    * Given a list of conditions in params and a list of desired
 705    * return Properties generate the required select and from
 706    * clauses. Note that since the where clause introduces new
 707    * tables, the initial attempt also retrieves all variables used
 708    * in the params list
 709    *
 710    * @param string $apiEntity
 711    *   The api entity being called.
 712    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 713    */
 714   public function selectClause($apiEntity = NULL) {
 715
 716     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 717     // param ($apiEntity) instead - presumably an oversight & 2 because
 718     // contact is not implemented as a component.
 719     $this->addSpecialFields($apiEntity);
 720
 721     foreach ($this->_fields as $name => $field) {
 722       // skip component fields
 723       // there are done by the alter query below
 724       // and need not be done on every field
 725       // @todo remove these & handle using metadata - only obscure fields
 726       // that are hack-added should need to be excluded from the main loop.
 727       if (
 728         (substr($name, 0, 12) === 'participant_') ||
 729         (substr($name, 0, 7) === 'pledge_') ||
 730         (substr($name, 0, 5) === 'case_')
 731       ) {
 732         continue;
 733       }
 734
 735       // redirect to activity select clause
 736       if (
 737         (substr($name, 0, 9) === 'activity_') ||
 738         ($name === 'parent_id')
 739       ) {
 740         CRM_Activity_BAO_Query::select($this);
 741       }
 742
 743       // if this is a hierarchical name, we ignore it
 744       $names = explode('-', $name);
 745       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 746         continue;
 747       }
 748
 749       // make an exception for special cases, to add the field in select clause
 750       $makeException = FALSE;
 751
 752       //special handling for groups/tags
 753       if (in_array($name, ['groups', 'tags', 'notes'])
 754         && isset($this->_returnProperties[substr($name, 0, -1)])
 755       ) {
 756         // @todo instead of setting make exception to get us into
 757         // an if clause that has handling for these fields buried with in it
 758         // move the handling to here.
 759         $makeException = TRUE;
 760       }
 761
 762       // since note has 3 different options we need special handling
 763       // note / note_subject / note_body
 764       if ($name === 'notes') {
 765         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 766           if (isset($this->_returnProperties[$noteField])) {
 767             $makeException = TRUE;
 768             break;
 769           }
 770         }
 771       }
 772
 773       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 774       if (
 775         !empty($this->_paramLookup[$name])
 776         || !empty($this->_returnProperties[$name])
 777         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 778         || $makeException
 779       ) {
 780         if ($cfID) {
 781           // add to cfIDs array if not present
 782           if (!array_key_exists($cfID, $this->_cfIDs)) {
 783             $this->_cfIDs[$cfID] = [];
 784           }
 785         }
 786         elseif (isset($field['where'])) {
 787           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 788           if (isset($tableName)) {
 789             if (!empty(self::$_dependencies[$tableName])) {
 790               $this->_tables['civicrm_address'] = 1;
 791               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 792               $this->_element['address_id'] = 1;
 793             }
 794
 795             if ($tableName === 'im_provider' || $tableName === 'email_greeting' ||
 796               $tableName === 'postal_greeting' || $tableName === 'addressee'
 797             ) {
 798               if ($tableName === 'im_provider') {
 799                 CRM_Core_OptionValue::select($this);
 800               }
 801
 802               if (in_array($tableName,
 803                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 804                 $this->_element["{$name}_id"] = 1;
 805                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 806                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 807                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 808                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 809
 810                 if ($tableName === 'email_greeting') {
 811                   // @todo bad join.
 812                   $this->_pseudoConstantsSelect[$name]['join']
 813                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 814                   $this->_pseudoConstantsSelect[$name]['join'] .=
 815                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 816                 }
 817                 elseif ($tableName === 'postal_greeting') {
 818                   // @todo bad join.
 819                   $this->_pseudoConstantsSelect[$name]['join']
 820                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 821                   $this->_pseudoConstantsSelect[$name]['join'] .=
 822                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 823                 }
 824                 elseif ($tableName == 'addressee') {
 825                   // @todo bad join.
 826                   $this->_pseudoConstantsSelect[$name]['join']
 827                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 828                   $this->_pseudoConstantsSelect[$name]['join'] .=
 829                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 830                 }
 831                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 832
 833                 //get display
 834                 $greetField = "{$name}_display";
 835                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 836                 $this->_element[$greetField] = 1;
 837                 //get custom
 838                 $greetField = "{$name}_custom";
 839                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 840                 $this->_element[$greetField] = 1;
 841               }
 842             }
 843             else {
 844               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 845                 $this->_tables[$tableName] = 1;
 846               }
 847
 848               // also get the id of the tableName
 849               $tName = substr($tableName, 8);
 850               if (in_array($tName, ['country', 'state_province', 'county'])) {
 851                 if ($tName == 'state_province') {
 852                   $this->_pseudoConstantsSelect['state_province_name'] = [
 853                     'pseudoField' => "{$tName}",
 854                     'idCol' => "{$tName}_id",
 855                     'bao' => 'CRM_Core_BAO_Address',
 856                     'table' => "civicrm_{$tName}",
 857                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 858                   ];
 859
 860                   $this->_pseudoConstantsSelect[$tName] = [
 861                     'pseudoField' => 'state_province_abbreviation',
 862                     'idCol' => "{$tName}_id",
 863                     'table' => "civicrm_{$tName}",
 864                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 865                   ];
 866                 }
 867                 else {
 868                   $this->_pseudoConstantsSelect[$name] = [
 869                     'pseudoField' => "{$tName}_id",
 870                     'idCol' => "{$tName}_id",
 871                     'bao' => 'CRM_Core_BAO_Address',
 872                     'table' => "civicrm_{$tName}",
 873                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 874                   ];
 875                 }
 876
 877                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 878                 $this->_element["{$tName}_id"] = 1;
 879               }
 880               elseif ($tName != 'contact') {
 881                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 882                 $this->_element["{$tName}_id"] = 1;
 883               }
 884
 885               //special case for phone
 886               if ($name == 'phone') {
 887                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 888                 $this->_element['phone_type_id'] = 1;
 889               }
 890
 891               // if IM then select provider_id also
 892               // to get "IM Service Provider" in a file to be exported, CRM-3140
 893               if ($name == 'im') {
 894                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 895                 $this->_element['provider_id'] = 1;
 896               }
 897
 898               if ($tName == 'contact' && $fieldName == 'organization_name') {
 899                 // special case, when current employer is set for Individual contact
 900                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 901               }
 902               elseif ($tName == 'contact' && $fieldName === 'id') {
 903                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 904               }
 905               elseif (in_array($tName, ['country', 'county'])) {
 906                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 907                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 908               }
 909               elseif ($tName == 'state_province') {
 910                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 911                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 912               }
 913               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 914                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 915                   $this->_select[$name] = "{$field['where']} as `$name`";
 916                 }
 917               }
 918               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 919                 $this->addPseudoconstantFieldToSelect($name);
 920               }
 921               else {
 922                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 923               }
 924               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 925                 $this->_element[$name] = 1;
 926               }
 927             }
 928           }
 929         }
 930         elseif ($name === 'tags') {
 931           //@todo move this handling outside the big IF & ditch $makeException
 932           $this->_useGroupBy = TRUE;
 933           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 934           $this->_element[$name] = 1;
 935           $this->_tables['civicrm_tag'] = 1;
 936           $this->_tables['civicrm_entity_tag'] = 1;
 937         }
 938         elseif ($name === 'groups') {
 939           //@todo move this handling outside the big IF & ditch $makeException
 940           $this->_useGroupBy = TRUE;
 941           // Duplicates will be created here but better to sort them out in php land.
 942           $this->_select[$name] = "
 943             CONCAT_WS(',',
 944             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 945             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 946           )
 947           as `groups`";
 948           $this->_element[$name] = 1;
 949           $this->_tables['civicrm_group_contact'] = 1;
 950           $this->_tables['civicrm_group_contact_cache'] = 1;
 951           $this->_pseudoConstantsSelect["{$name}"] = [
 952             'pseudoField' => "groups",
 953             'idCol' => 'groups',
 954           ];
 955         }
 956         elseif ($name === 'notes') {
 957           //@todo move this handling outside the big IF & ditch $makeException
 958           // if note field is subject then return subject else body of the note
 959           $noteColumn = 'note';
 960           if (isset($noteField) && $noteField === 'note_subject') {
 961             $noteColumn = 'subject';
 962           }
 963
 964           $this->_useGroupBy = TRUE;
 965           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 966           $this->_element[$name] = 1;
 967           $this->_tables['civicrm_note'] = 1;
 968         }
 969         elseif ($name === 'current_employer') {
 970           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 971           $this->_element[$name] = 1;
 972         }
 973       }
 974
 975       if ($cfID && !empty($field['is_search_range'])) {
 976         // this is a custom field with range search enabled, so we better check for two/from values
 977         if (!empty($this->_paramLookup[$name . '_from'])) {
 978           if (!array_key_exists($cfID, $this->_cfIDs)) {
 979             $this->_cfIDs[$cfID] = [];
 980           }
 981           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 982             // search in the cdID array for the same grouping
 983             $fnd = FALSE;
 984             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 985               if ($c[3] == $p[3]) {
 986                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 987                 $fnd = TRUE;
 988               }
 989             }
 990             if (!$fnd) {
 991               $p[2] = ['from' => $p[2]];
 992               $this->_cfIDs[$cfID][] = $p;
 993             }
 994           }
 995         }
 996         if (!empty($this->_paramLookup[$name . '_to'])) {
 997           if (!array_key_exists($cfID, $this->_cfIDs)) {
 998             $this->_cfIDs[$cfID] = [];
 999           }
1000           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
1001             // search in the cdID array for the same grouping
1002             $fnd = FALSE;
1003             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
1004               if ($c[4] == $p[4]) {
1005                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
1006                 $fnd = TRUE;
1007               }
1008             }
1009             if (!$fnd) {
1010               $p[2] = ['to' => $p[2]];
1011               $this->_cfIDs[$cfID][] = $p;
1012             }
1013           }
1014         }
1015       }
1016     }
1017
1018     // add location as hierarchical elements
1019     $this->addHierarchicalElements();
1020
1021     // add multiple field like website
1022     $this->addMultipleElements();
1023
1024     //fix for CRM-951
1025     CRM_Core_Component::alterQuery($this, 'select');
1026
1027     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1028   }
1029
1030   /**
1031    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1032    */
1033   public function addHierarchicalElements() {
1034     if (empty($this->_returnProperties['location'])) {
1035       return;
1036     }
1037     if (!is_array($this->_returnProperties['location'])) {
1038       return;
1039     }
1040
1041     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1042     $processed = [];
1043     $index = 0;
1044
1045     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1046     $addressCustomFieldIds = [];
1047
1048     foreach ($this->_returnProperties['location'] as $name => $elements) {
1049       $lCond = self::getPrimaryCondition($name);
1050       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1051
1052       if (!$lCond) {
1053         if ($locationTypeId === FALSE) {
1054           continue;
1055         }
1056         $lCond = "location_type_id = $locationTypeId";
1057         $this->_useDistinct = TRUE;
1058
1059         //commented for CRM-3256
1060         $this->_useGroupBy = TRUE;
1061       }
1062
1063       $name = str_replace(' ', '_', $name);
1064       $tName = "$name-location_type";
1065       $ltName = "`$name-location_type`";
1066       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1067       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1068       $this->_element["{$tName}_id"] = 1;
1069       $this->_element["{$tName}"] = 1;
1070
1071       $locationTypeName = $tName;
1072       $locationTypeJoin = [];
1073
1074       $addWhereCount = 0;
1075       foreach ($elements as $elementFullName => $dontCare) {
1076         $index++;
1077         $elementName = $elementCmpName = $elementFullName;
1078
1079         if (substr($elementCmpName, 0, 5) == 'phone') {
1080           $elementCmpName = 'phone';
1081         }
1082
1083         if (array_key_exists($elementCmpName, $addressCustomFields)) {
1084           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1085             $addressCustomFieldIds[$cfID][$name] = 1;
1086           }
1087         }
1088         // add address table - doesn't matter if we do it mutliple times - it's the same data
1089         // @todo ditch the double processing of addressJoin
1090         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1091           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1092         ) {
1093           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1094           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1095           $processed[$aName] = 1;
1096         }
1097
1098         $cond = $elementType = '';
1099         if (strpos($elementName, '-') !== FALSE) {
1100           // this is either phone, email or IM
1101           list($elementName, $elementType) = explode('-', $elementName);
1102
1103           if (($elementName != 'phone') && ($elementName != 'im')) {
1104             $cond = self::getPrimaryCondition($elementType);
1105           }
1106           // CRM-13011 : If location type is primary, do not restrict search to the phone
1107           // type id - we want the primary phone, regardless of what type it is.
1108           // Otherwise, restrict to the specified phone type for the given field.
1109           if ((!$cond) && ($elementName == 'phone')) {
1110             $cond = "phone_type_id = '$elementType'";
1111           }
1112           elseif ((!$cond) && ($elementName == 'im')) {
1113             // IM service provider id, CRM-3140
1114             $cond = "provider_id = '$elementType'";
1115           }
1116           $elementType = '-' . $elementType;
1117         }
1118
1119         $field = $this->_fields[$elementName] ?? NULL;
1120         if (!empty($field)) {
1121           if (isset($this->_pseudoConstantsSelect[$field['name']])) {
1122             $this->_pseudoConstantsSelect[$name . '-' . $field['name']] = $this->_pseudoConstantsSelect[$field['name']];
1123           }
1124         }
1125
1126         // hack for profile, add location id
1127         if (!$field) {
1128           if ($elementType &&
1129             // fix for CRM-882( to handle phone types )
1130             !is_numeric($elementType)
1131           ) {
1132             if (is_numeric($name)) {
1133               $field = $this->_fields[$elementName . "-Primary$elementType"] ?? NULL;
1134             }
1135             else {
1136               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1137             }
1138           }
1139           elseif (is_numeric($name)) {
1140             //this for phone type to work
1141             if (in_array($elementName, ['phone', 'phone_ext'])) {
1142               $field = $this->_fields[$elementName . "-Primary" . $elementType] ?? NULL;
1143             }
1144             else {
1145               $field = $this->_fields[$elementName . "-Primary"] ?? NULL;
1146             }
1147           }
1148           else {
1149             //this is for phone type to work for profile edit
1150             if (in_array($elementName, ['phone', 'phone_ext'])) {
1151               $field = $this->_fields[$elementName . "-$locationTypeId$elementType"] ?? NULL;
1152             }
1153             else {
1154               $field = $this->_fields[$elementName . "-$locationTypeId"] ?? NULL;
1155             }
1156           }
1157         }
1158
1159         // Check if there is a value, if so also add to where Clause
1160         $addWhere = FALSE;
1161         if ($this->_params) {
1162           $nm = $elementName;
1163           if (isset($locationTypeId)) {
1164             $nm .= "-$locationTypeId";
1165           }
1166           if (!is_numeric($elementType)) {
1167             $nm .= "$elementType";
1168           }
1169
1170           foreach ($this->_params as $id => $values) {
1171             if ((is_array($values) && $values[0] == $nm) ||
1172               (in_array($elementName, ['phone', 'im'])
1173                 && (strpos($values[0], $nm) !== FALSE)
1174               )
1175             ) {
1176               $addWhere = TRUE;
1177               $addWhereCount++;
1178               break;
1179             }
1180           }
1181         }
1182
1183         if ($field && isset($field['where'])) {
1184           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1185           $pf = substr($tableName, 8);
1186           $tName = $name . '-' . $pf . $elementType;
1187           if (isset($tableName)) {
1188             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1189               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1190             }
1191             else {
1192               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1193             }
1194
1195             $this->_element["{$tName}_id"] = 1;
1196             if (substr($tName, -15) == '-state_province') {
1197               // FIXME: hack to fix CRM-1900
1198               $a = Civi::settings()->get('address_format');
1199
1200               if (substr_count($a, 'state_province_name') > 0) {
1201                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1202                   'pseudoField' => "{$pf}_id",
1203                   'idCol' => "{$tName}_id",
1204                   'bao' => 'CRM_Core_BAO_Address',
1205                 ];
1206                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1207               }
1208               else {
1209                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1210                   'pseudoField' => 'state_province_abbreviation',
1211                   'idCol' => "{$tName}_id",
1212                 ];
1213                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1214               }
1215             }
1216             else {
1217               if (substr($elementFullName, 0, 2) == 'im') {
1218                 $provider = "{$name}-{$elementFullName}-provider_id";
1219                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1220                 $this->_element[$provider] = 1;
1221               }
1222               if ($pf == 'country' || $pf == 'county') {
1223                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1224                   'pseudoField' => "{$pf}_id",
1225                   'idCol' => "{$tName}_id",
1226                   'bao' => 'CRM_Core_BAO_Address',
1227                 ];
1228                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1229               }
1230               else {
1231                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1232               }
1233             }
1234
1235             if (in_array($pf, ['state_province', 'country', 'county'])) {
1236               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1237             }
1238             else {
1239               $this->_element["{$name}-{$elementFullName}"] = 1;
1240             }
1241
1242             if (empty($processed["`$tName`"])) {
1243               $processed["`$tName`"] = 1;
1244               $newName = $tableName . '_' . $index;
1245               switch ($tableName) {
1246                 case 'civicrm_phone':
1247                 case 'civicrm_email':
1248                 case 'civicrm_im':
1249                 case 'civicrm_openid':
1250
1251                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1252                   if ($tableName != 'civicrm_phone') {
1253                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1254                   }
1255                   elseif (is_numeric($name)) {
1256                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1257                   }
1258
1259                   // this special case to add phone type
1260                   if ($cond) {
1261                     $phoneTypeCondition = " AND `$tName`.$cond ";
1262                     //gross hack to pickup corrupted data also, CRM-7603
1263                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1264                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1265                       if (!empty($lCond)) {
1266                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1267                       }
1268                     }
1269                     $this->_tables[$tName] .= $phoneTypeCondition;
1270                   }
1271
1272                   //build locationType join
1273                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1274
1275                   if ($addWhere) {
1276                     $this->_whereTables[$tName] = $this->_tables[$tName];
1277                   }
1278                   break;
1279
1280                 case 'civicrm_state_province':
1281                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1282                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1283                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1284                   if ($addWhere) {
1285                     $this->_whereTables["{$name}-address"] = $addressJoin;
1286                   }
1287                   break;
1288
1289                 case 'civicrm_location_type':
1290                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.location_type_id";
1291
1292                   if ($addWhere) {
1293                     $this->_whereTables["{$name}-address"] = $addressJoin;
1294                   }
1295                   break;
1296
1297                 case 'civicrm_country':
1298                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1299                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1300                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1301                   if ($addWhere) {
1302                     $this->_whereTables["{$name}-address"] = $addressJoin;
1303                   }
1304                   break;
1305
1306                 case 'civicrm_county':
1307                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1308                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1309                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1310                   if ($addWhere) {
1311                     $this->_whereTables["{$name}-address"] = $addressJoin;
1312                   }
1313                   break;
1314
1315                 default:
1316                   if (isset($addressCustomFields[$elementName]['custom_field_id']) && !empty($addressCustomFields[$elementName]['custom_field_id'])) {
1317                     $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.id";
1318                   }
1319                   if ($addWhere) {
1320                     $this->_whereTables["{$name}-address"] = $addressJoin;
1321                   }
1322                   break;
1323               }
1324             }
1325           }
1326         }
1327       }
1328
1329       // add location type  join
1330       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1331       $this->_tables[$locationTypeName] = $ltypeJoin;
1332
1333       // table should be present in $this->_whereTables,
1334       // to add its condition in location type join, CRM-3939.
1335       if ($addWhereCount) {
1336         $locClause = [];
1337         foreach ($this->_whereTables as $tableName => $clause) {
1338           if (!empty($locationTypeJoin[$tableName])) {
1339             $locClause[] = $locationTypeJoin[$tableName];
1340           }
1341         }
1342
1343         if (!empty($locClause)) {
1344           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1345         }
1346       }
1347     }
1348
1349     if (!empty($addressCustomFieldIds)) {
1350       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1351       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1352         foreach ($locTypeName as $name => $dnc) {
1353           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1354           $fieldName = "$name-custom_{$cfID}";
1355           $tName = "$name-address-custom-{$cfID}";
1356           $aName = "`$name-address-custom-{$cfID}`";
1357           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1358           $this->_element["{$tName}_id"] = 1;
1359           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1360           $this->_element[$fieldName] = 1;
1361           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1362         }
1363       }
1364     }
1365   }
1366
1367   /**
1368    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1369    */
1370   public function addMultipleElements() {
1371     if (empty($this->_returnProperties['website'])) {
1372       return;
1373     }
1374     if (!is_array($this->_returnProperties['website'])) {
1375       return;
1376     }
1377
1378     foreach ($this->_returnProperties['website'] as $key => $elements) {
1379       foreach ($elements as $elementFullName => $dontCare) {
1380         $tName = "website-{$key}-{$elementFullName}";
1381         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1382         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1383         $this->_element["{$tName}_id"] = 1;
1384         $this->_element["{$tName}"] = 1;
1385
1386         $type = "website-{$key}-website_type_id";
1387         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1388         $this->_element[$type] = 1;
1389         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1390       }
1391     }
1392   }
1393
1394   /**
1395    * Generate the query based on what type of query we need.
1396    *
1397    * @param bool $count
1398    * @param bool $sortByChar
1399    * @param bool $groupContacts
1400    * @param bool $onlyDeleted
1401    *
1402    * @return array
1403    *   sql query parts as an array
1404    */
1405   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1406     // build permission clause
1407     $this->generatePermissionClause($onlyDeleted, $count);
1408
1409     if ($count) {
1410       if (isset($this->_rowCountClause)) {
1411         $select = "SELECT {$this->_rowCountClause}";
1412       }
1413       elseif (isset($this->_distinctComponentClause)) {
1414         // we add distinct to get the right count for components
1415         // for the more complex result set, we use GROUP BY the same id
1416         // CRM-9630
1417         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1418       }
1419       else {
1420         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1421       }
1422       $from = $this->_simpleFromClause;
1423       if ($this->_useDistinct) {
1424         $this->_useGroupBy = TRUE;
1425       }
1426     }
1427     elseif ($sortByChar) {
1428       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1429       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1430       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1431       $from = $this->_simpleFromClause;
1432     }
1433     elseif ($groupContacts) {
1434       $select = 'SELECT contact_a.id as id';
1435       if ($this->_useDistinct) {
1436         $this->_useGroupBy = TRUE;
1437       }
1438       $from = $this->_simpleFromClause;
1439     }
1440     else {
1441       if (!empty($this->_paramLookup['group'])) {
1442
1443         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1444
1445         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1446           $this->_paramLookup['group'][0][1] = key($value);
1447         }
1448
1449         // Presumably the lines below come into manage groups screen.
1450         // make sure there is only one element
1451         // this is used when we are running under smog and need to know
1452         // how the contact was added (CRM-1203)
1453         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1454         if ((count($this->_paramLookup['group']) == 1) &&
1455           (count($groups) == 1)
1456         ) {
1457           $groupId = $groups[0];
1458
1459           //check if group is saved search
1460           $group = new CRM_Contact_BAO_Group();
1461           $group->id = $groupId;
1462           $group->find(TRUE);
1463
1464           if (!isset($group->saved_search_id)) {
1465             $tbName = "civicrm_group_contact";
1466             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1467             // as this will add load to an intentionally light query.
1468             // ideally this code would be removed as it appears to be to support CRM-1203
1469             // and passing in the required returnProperties from the url would
1470             // make more sense that globally applying the requirements of one form.
1471             if (($this->_returnProperties != ['contact_id'])) {
1472               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1473               $this->_element['group_contact_id'] = 1;
1474               $this->_select['status'] = "$tbName.status as status";
1475               $this->_element['status'] = 1;
1476             }
1477           }
1478         }
1479         $this->_useGroupBy = TRUE;
1480       }
1481       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1482         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1483           // CRM-5954
1484           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1485           $this->_useDistinct = FALSE;
1486           $this->_useGroupBy = TRUE;
1487         }
1488       }
1489
1490       $select = $this->getSelect();
1491       $from = $this->_fromClause;
1492     }
1493
1494     $where = '';
1495     if (!empty($this->_whereClause)) {
1496       $where = "WHERE {$this->_whereClause}";
1497     }
1498
1499     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1500       if (!empty($this->_permissionFromClause)) {
1501         $from .= " $this->_permissionFromClause";
1502       }
1503       if (empty($where)) {
1504         $where = "WHERE $this->_permissionWhereClause";
1505       }
1506       else {
1507         $where = "$where AND $this->_permissionWhereClause";
1508       }
1509     }
1510
1511     $having = '';
1512     if (!empty($this->_having)) {
1513       foreach ($this->_having as $havingSets) {
1514         foreach ($havingSets as $havingSet) {
1515           $havingValue[] = $havingSet;
1516         }
1517       }
1518       $having = ' HAVING ' . implode(' AND ', $havingValue);
1519     }
1520
1521     // if we are doing a transform, do it here
1522     // use the $from, $where and $having to get the contact ID
1523     if ($this->_displayRelationshipType) {
1524       $this->filterRelatedContacts($from, $where, $having);
1525     }
1526
1527     return [$select, $from, $where, $having];
1528   }
1529
1530   /**
1531    * Get where values from the parameters.
1532    *
1533    * @param string $name
1534    * @param mixed $grouping
1535    *
1536    * @return mixed
1537    */
1538   public function getWhereValues($name, $grouping) {
1539     $result = NULL;
1540     foreach ($this->_params as $values) {
1541       if ($values[0] == $name && $values[3] == $grouping) {
1542         return $values;
1543       }
1544     }
1545
1546     return $result;
1547   }
1548
1549   /**
1550    * Fix date values.
1551    *
1552    * @param bool $relative
1553    * @param string $from
1554    * @param string $to
1555    */
1556   public static function fixDateValues($relative, &$from, &$to) {
1557     if ($relative) {
1558       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1559     }
1560   }
1561
1562   /**
1563    * Convert values from form-appropriate to query-object appropriate.
1564    *
1565    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1566    * So, ideally we would convert all fields to look like
1567    *  array(
1568    *   0 => $fieldName
1569    *   // Set the operator for legacy reasons, but it is ignored
1570    *   1 =>  '='
1571    *   // array in sql filter syntax
1572    *   2 => array('BETWEEN' => array(1,60),
1573    *   3 => null
1574    *   4 => null
1575    *  );
1576    *
1577    * There are some examples of the syntax in
1578    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1579    *
1580    * More notes at CRM_Core_DAO::createSQLFilter
1581    *
1582    * and a list of supported operators in CRM_Core_DAO
1583    *
1584    * @param array $formValues
1585    * @param int $wildcard
1586    * @param bool $useEquals
1587    *
1588    * @param string $apiEntity
1589    *
1590    * @param array $entityReferenceFields
1591    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1592    *
1593    * @return array
1594    */
1595   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1596     $entityReferenceFields = []) {
1597     $params = [];
1598     if (empty($formValues)) {
1599       return $params;
1600     }
1601
1602     self::filterCountryFromValuesIfStateExists($formValues);
1603     CRM_Core_BAO_CustomValue::fixCustomFieldValue($formValues);
1604
1605     foreach ($formValues as $id => $values) {
1606       if (self::isAlreadyProcessedForQueryFormat($values)) {
1607         $params[] = $values;
1608         continue;
1609       }
1610
1611       self::legacyConvertFormValues($id, $values);
1612
1613       // The form uses 1 field to represent two db fields
1614       if ($id === 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1615         $contactType = [];
1616         $subType = [];
1617         foreach ((array) $values as $key => $type) {
1618           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1619           $contactType[$types[0]] = $types[0];
1620           // Add sub-type if specified
1621           if (!empty($types[1])) {
1622             $subType[$types[1]] = $types[1];
1623           }
1624         }
1625         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1626         if ($subType) {
1627           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1628         }
1629       }
1630       elseif ($id === 'privacy') {
1631         if (is_array($formValues['privacy'])) {
1632           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1633           foreach ($formValues['privacy'] as $key => $value) {
1634             if ($value) {
1635               $params[] = [$key, $op, $value, 0, 0];
1636             }
1637           }
1638         }
1639       }
1640       elseif ($id === 'email_on_hold') {
1641         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1642           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1643           // so in 5.11 we have an extra if that should become redundant over time.
1644           // https://lab.civicrm.org/dev/core/issues/745
1645           // @todo this renaming of email_on_hold to on_hold needs revisiting
1646           // it precedes recent changes but causes the default not to reload.
1647           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1648           if (!empty($onHoldValue)) {
1649             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1650           }
1651         }
1652       }
1653       elseif (substr($id, 0, 7) === 'custom_'
1654         &&  (
1655           substr($id, -5, 5) === '_from'
1656           || substr($id, -3, 3) === '_to'
1657         )
1658       ) {
1659         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1660       }
1661       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1662         FALSE)) {
1663         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1664       }
1665       else {
1666         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1667
1668         if (!$values) {
1669           continue;
1670         }
1671         $params[] = $values;
1672       }
1673     }
1674     return $params;
1675   }
1676
1677   /**
1678    * Function to support legacy format for groups and tags.
1679    *
1680    * @param string $id
1681    * @param array|int $values
1682    *
1683    */
1684   public static function legacyConvertFormValues($id, &$values) {
1685     $legacyElements = [
1686       'group',
1687       'tag',
1688       'contact_tags',
1689       'contact_type',
1690       'membership_type_id',
1691       'membership_status_id',
1692     ];
1693     if (in_array($id, $legacyElements) && is_array($values)) {
1694       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1695       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1696       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1697     }
1698   }
1699
1700   /**
1701    * Fix values from query from/to something no-one cared enough to document.
1702    *
1703    * @param int $id
1704    * @param array $values
1705    * @param int $wildcard
1706    * @param bool $useEquals
1707    *
1708    * @param string $apiEntity
1709    *
1710    * @return array|null
1711    */
1712   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1713     // skip a few search variables
1714     static $skipWhere = NULL;
1715     static $likeNames = NULL;
1716     $result = NULL;
1717
1718     // Change camelCase EntityName to lowercase with underscores
1719     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1720
1721     // check if $value is in OK (Operator as Key) format as used by Get API
1722     if (CRM_Utils_System::isNull($values)) {
1723       return $result;
1724     }
1725
1726     if (!$skipWhere) {
1727       $skipWhere = [
1728         'task',
1729         'radio_ts',
1730         'uf_group_id',
1731         'component_mode',
1732         'qfKey',
1733         'operator',
1734         'display_relationship_type',
1735       ];
1736     }
1737
1738     if (in_array($id, $skipWhere) ||
1739       substr($id, 0, 4) == '_qf_' ||
1740       substr($id, 0, 7) == 'hidden_'
1741     ) {
1742       return $result;
1743     }
1744
1745     if ($apiEntity &&
1746       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1747       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1748       (substr($id, 0, 7) != 'custom_')
1749     ) {
1750       $id = $apiEntity . '_' . $id;
1751     }
1752
1753     if (!$likeNames) {
1754       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1755     }
1756
1757     // email comes in via advanced search
1758     // so use wildcard always
1759     if ($id == 'email') {
1760       $wildcard = 1;
1761     }
1762
1763     if (!$useEquals && in_array($id, $likeNames)) {
1764       $result = [$id, 'LIKE', $values, 0, 1];
1765     }
1766     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1767       $result = [$id, 'LIKE', $values, 0, 0];
1768     }
1769     elseif ($id == 'contact_type' ||
1770       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1771     ) {
1772       $result = [$id, 'IN', $values, 0, $wildcard];
1773     }
1774     else {
1775       $result = [$id, '=', $values, 0, $wildcard];
1776     }
1777
1778     return $result;
1779   }
1780
1781   /**
1782    * Get the where clause for a single field.
1783    *
1784    * @param array $values
1785    * @param bool $isForcePrimaryOnly
1786    *
1787    * @throws \CRM_Core_Exception
1788    */
1789   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1790     if ($this->isARelativeDateField($values[0])) {
1791       $this->buildRelativeDateQuery($values);
1792       return;
1793     }
1794     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1795
1796     // do not process custom fields or prefixed contact ids or component params
1797     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1798       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1799       (substr($values[0], 0, 13) === 'contribution_') ||
1800       (substr($values[0], 0, 6) === 'event_') ||
1801       (substr($values[0], 0, 12) === 'participant_') ||
1802       (substr($values[0], 0, 7) === 'member_') ||
1803       (substr($values[0], 0, 6) === 'grant_') ||
1804       (substr($values[0], 0, 7) === 'pledge_') ||
1805       (substr($values[0], 0, 5) === 'case_') ||
1806       (substr($values[0], 0, 10) === 'financial_') ||
1807       (substr($values[0], 0, 8) === 'payment_') ||
1808       (substr($values[0], 0, 11) === 'membership_')
1809       // temporary fix for regression https://lab.civicrm.org/dev/core/issues/1551
1810       //  ideally the metadata would allow  this field to be parsed below & the special handling would not
1811       // be needed.
1812       || $values[0] === 'mailing_id'
1813     ) {
1814       return;
1815     }
1816
1817     // skip for hook injected fields / params
1818     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1819     if (array_key_exists($values[0], $extFields)) {
1820       return;
1821     }
1822
1823     switch ($values[0]) {
1824       case 'deleted_contacts':
1825         $this->deletedContacts($values);
1826         return;
1827
1828       case 'contact_sub_type':
1829         $this->contactSubType($values);
1830         return;
1831
1832       case 'group':
1833       case 'group_type':
1834         $this->group($values);
1835         return;
1836
1837       // case tag comes from find contacts
1838       case 'tag_search':
1839         $this->tagSearch($values);
1840         return;
1841
1842       case 'tag':
1843       case 'contact_tags':
1844         $this->tag($values);
1845         return;
1846
1847       case 'note':
1848       case 'note_body':
1849       case 'note_subject':
1850         $this->notes($values);
1851         return;
1852
1853       case 'uf_user':
1854         $this->ufUser($values);
1855         return;
1856
1857       case 'sort_name':
1858       case 'display_name':
1859         $this->sortName($values);
1860         return;
1861
1862       case 'addressee':
1863       case 'postal_greeting':
1864       case 'email_greeting':
1865         $this->greetings($values);
1866         return;
1867
1868       case 'email':
1869       case 'email_id':
1870         $this->email($values, $isForcePrimaryOnly);
1871         return;
1872
1873       case 'phone_numeric':
1874         $this->phone_numeric($values);
1875         return;
1876
1877       case 'phone_phone_type_id':
1878       case 'phone_location_type_id':
1879         $this->phone_option_group($values);
1880         return;
1881
1882       case 'street_address':
1883         $this->street_address($values);
1884         return;
1885
1886       case 'street_number':
1887         $this->street_number($values);
1888         return;
1889
1890       case 'sortByCharacter':
1891         $this->sortByCharacter($values);
1892         return;
1893
1894       case 'location_type':
1895         $this->locationType($values);
1896         return;
1897
1898       case 'county':
1899         $this->county($values);
1900         return;
1901
1902       case 'state_province':
1903       case 'state_province_id':
1904       case 'state_province_name':
1905         $this->stateProvince($values);
1906         return;
1907
1908       case 'country':
1909       case 'country_id':
1910         $this->country($values, FALSE);
1911         return;
1912
1913       case 'postal_code':
1914       case 'postal_code_low':
1915       case 'postal_code_high':
1916         $this->postalCode($values);
1917         return;
1918
1919       case 'activity_date':
1920       case 'activity_date_low':
1921       case 'activity_date_high':
1922       case 'activity_date_time_low':
1923       case 'activity_date_time_high':
1924       case 'activity_role':
1925       case 'activity_status_id':
1926       case 'activity_status':
1927       case 'activity_priority':
1928       case 'activity_priority_id':
1929       case 'followup_parent_id':
1930       case 'parent_id':
1931       case 'source_contact_id':
1932       case 'activity_text':
1933       case 'activity_option':
1934       case 'test_activities':
1935       case 'activity_type_id':
1936       case 'activity_type':
1937       case 'activity_survey_id':
1938       case 'activity_tags':
1939       case 'activity_taglist':
1940       case 'activity_test':
1941       case 'activity_campaign_id':
1942       case 'activity_engagement_level':
1943       case 'activity_id':
1944       case 'activity_result':
1945       case 'source_contact':
1946         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1947         return;
1948
1949       case 'age_low':
1950       case 'age_high':
1951       case 'birth_date_low':
1952       case 'birth_date_high':
1953       case 'deceased_date_low':
1954       case 'deceased_date_high':
1955         $this->demographics($values);
1956         return;
1957
1958       case 'age_asof_date':
1959         // handled by demographics
1960         return;
1961
1962       case 'log_date_low':
1963       case 'log_date_high':
1964         $this->modifiedDates($values);
1965         return;
1966
1967       case 'changed_by':
1968         $this->changeLog($values);
1969         return;
1970
1971       case 'do_not_phone':
1972       case 'do_not_email':
1973       case 'do_not_mail':
1974       case 'do_not_sms':
1975       case 'do_not_trade':
1976       case 'is_opt_out':
1977         $this->privacy($values);
1978         return;
1979
1980       case 'privacy_options':
1981         $this->privacyOptions($values);
1982         return;
1983
1984       case 'privacy_operator':
1985       case 'privacy_toggle':
1986         // these are handled by privacy options
1987         return;
1988
1989       case 'preferred_communication_method':
1990         $this->preferredCommunication($values);
1991         return;
1992
1993       case 'relation_type_id':
1994       case 'relationship_start_date_high':
1995       case 'relationship_start_date_low':
1996       case 'relationship_end_date_high':
1997       case 'relationship_end_date_low':
1998       case 'relation_active_period_date_high':
1999       case 'relation_active_period_date_low':
2000       case 'relation_target_name':
2001       case 'relation_status':
2002       case 'relation_description':
2003       case 'relation_date_low':
2004       case 'relation_date_high':
2005         $this->relationship($values);
2006         $this->_relationshipValuesAdded = TRUE;
2007         return;
2008
2009       case 'task_status_id':
2010         $this->task($values);
2011         return;
2012
2013       case 'task_id':
2014         // since this case is handled with the above
2015         return;
2016
2017       case 'prox_distance':
2018         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2019         return;
2020
2021       case 'prox_street_address':
2022       case 'prox_city':
2023       case 'prox_postal_code':
2024       case 'prox_state_province_id':
2025       case 'prox_country_id':
2026       case 'prox_geo_code_1':
2027       case 'prox_geo_code_2':
2028         // handled by the proximity_distance clause
2029         return;
2030
2031       default:
2032         $this->restWhere($values);
2033         return;
2034     }
2035   }
2036
2037   /**
2038    * Given a list of conditions in params generate the required where clause.
2039    *
2040    * @param bool $isForcePrimaryEmailOnly
2041    *
2042    * @return string
2043    * @throws \CRM_Core_Exception
2044    */
2045   public function whereClause($isForcePrimaryEmailOnly = NULL): string {
2046     $this->_where[0] = [];
2047     $this->_qill[0] = [];
2048
2049     $this->includeContactIDs();
2050     if (!empty($this->_params)) {
2051       foreach (array_keys($this->_params) as $id) {
2052         if (empty($this->_params[$id][0])) {
2053           continue;
2054         }
2055         // check for both id and contact_id
2056         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2057           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2058           $field = $this->_fields['id'] ?? NULL;
2059           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2060             'CRM_Contact_BAO_Contact',
2061             "contact_a.id",
2062             $this->_params[$id][2],
2063             $this->_params[$id][1]
2064           );
2065           $this->_qill[0][] = ts("%1 %2 %3", [
2066             1 => $field['title'] ?? '',
2067             2 => $qillop,
2068             3 => $qillVal,
2069           ]);
2070         }
2071         else {
2072           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2073         }
2074       }
2075
2076       CRM_Core_Component::alterQuery($this, 'where');
2077
2078       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2079     }
2080
2081     if ($this->_customQuery) {
2082       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
2083       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2084       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2085       if (!empty($this->_customQuery->_where)) {
2086         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2087       }
2088       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2089     }
2090
2091     $clauses = [];
2092     $andClauses = [];
2093
2094     $validClauses = 0;
2095     if (!empty($this->_where)) {
2096       foreach ($this->_where as $grouping => $values) {
2097         if ($grouping > 0 && !empty($values)) {
2098           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2099           $validClauses++;
2100         }
2101       }
2102
2103       if (!empty($this->_where[0])) {
2104         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2105       }
2106       if (!empty($clauses)) {
2107         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2108       }
2109
2110       if ($validClauses > 1) {
2111         $this->_useDistinct = TRUE;
2112       }
2113     }
2114
2115     return implode(' AND ', $andClauses);
2116   }
2117
2118   /**
2119    * Generate where clause for any parameters not already handled.
2120    *
2121    * @param array $values
2122    *
2123    * @throws Exception
2124    */
2125   public function restWhere(&$values) {
2126     $name = $values[0] ?? NULL;
2127     $op = $values[1] ?? NULL;
2128     $value = $values[2] ?? NULL;
2129     $grouping = $values[3] ?? NULL;
2130     $wildcard = $values[4] ?? NULL;
2131
2132     if (isset($grouping) && empty($this->_where[$grouping])) {
2133       $this->_where[$grouping] = [];
2134     }
2135
2136     $multipleFields = ['url'];
2137
2138     //check if the location type exists for fields
2139     $lType = '';
2140     $locType = explode('-', $name);
2141
2142     if (!in_array($locType[0], $multipleFields)) {
2143       //add phone type if exists
2144       if (isset($locType[2]) && $locType[2]) {
2145         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2146       }
2147     }
2148
2149     $field = $this->_fields[$name] ?? NULL;
2150
2151     if (!$field) {
2152       $field = $this->_fields[$locType[0]] ?? NULL;
2153
2154       if (!$field) {
2155         // Strip any trailing _high & _low that might be appended.
2156         $realFieldName = str_replace(['_high', '_low'], '', $name);
2157         if (isset($this->_fields[$realFieldName])) {
2158           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2159           $columnName = $field['column_name'] ?? $field['name'];
2160           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $columnName, $field['title']);
2161         }
2162         return;
2163       }
2164     }
2165
2166     $setTables = TRUE;
2167
2168     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2169     if (isset($locType[1]) && is_numeric($locType[1])) {
2170       $lType = $locationType[$locType[1]];
2171     }
2172     if ($lType) {
2173       $field['title'] .= " ($lType)";
2174     }
2175
2176     if (substr($name, 0, 14) === 'state_province') {
2177       if (isset($locType[1]) && is_numeric($locType[1])) {
2178         $setTables = FALSE;
2179         $aName = "{$lType}-address";
2180         $where = "`$aName`.state_province_id";
2181       }
2182       else {
2183         $where = "civicrm_address.state_province_id";
2184       }
2185
2186       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2187       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2188       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2189       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2190     }
2191     elseif (!empty($field['pseudoconstant'])) {
2192       // For the hacked fields we want to undo the hack to type to avoid missing the index by adding quotes.
2193       $dataType = !empty($this->legacyHackedFields[$name]) ? CRM_Utils_Type::T_INT : $field['type'];
2194       $this->optionValueQuery(
2195         $name, $op, $value, $grouping,
2196         'CRM_Contact_DAO_Contact',
2197         $field,
2198         $field['html']['label'] ?? $field['title'],
2199         CRM_Utils_Type::typeToString($dataType)
2200       );
2201       if ($name === 'gender_id') {
2202         self::$_openedPanes[ts('Demographics')] = TRUE;
2203       }
2204     }
2205     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2206       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2207       if (isset($locType[1]) && is_numeric($locType[1])) {
2208         $setTables = FALSE;
2209         $aName = "{$lType}-address";
2210         $where = "`$aName`.$name";
2211       }
2212       else {
2213         $where = "civicrm_address.$name";
2214       }
2215
2216       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2217       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2218
2219       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2220       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2221     }
2222     elseif ($name === 'world_region') {
2223       $this->optionValueQuery(
2224         $name, $op, $value, $grouping,
2225         NULL,
2226         $field,
2227         ts('World Region'),
2228         'Positive'
2229       );
2230     }
2231     elseif ($name === 'is_deceased') {
2232       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2233       self::$_openedPanes[ts('Demographics')] = TRUE;
2234     }
2235     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2236       $appendDateTime = TRUE;
2237       if ($name === 'deceased_date' || $name === 'birth_date') {
2238         $appendDateTime = FALSE;
2239         self::$_openedPanes[ts('Demographics')] = TRUE;
2240       }
2241       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2242     }
2243     elseif ($name === 'contact_id') {
2244       if (is_int($value)) {
2245         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2246         $this->_qill[$grouping][] = "$field[title] $op $value";
2247       }
2248     }
2249     elseif ($name === 'name') {
2250       $value = CRM_Core_DAO::escapeString($value);
2251       if ($wildcard) {
2252         $op = 'LIKE';
2253         $value = self::getWildCardedValue($wildcard, $op, $value);
2254       }
2255       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2256       // @todo it's likely this code path is obsolete / never called. It is definitely not
2257       // passed through in our test suite.
2258       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2259       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2260     }
2261     elseif ($name === 'current_employer') {
2262       if ($wildcard) {
2263         $op = 'LIKE';
2264         $value = self::getWildCardedValue($wildcard, $op, $value);
2265       }
2266       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2267         $value
2268       );
2269       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2270       $this->_where[$grouping][] = $ceWhereClause;
2271       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2272     }
2273     elseif (substr($name, 0, 4) === 'url-') {
2274       $tName = 'civicrm_website';
2275       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2276       $value = CRM_Core_DAO::escapeString($value);
2277       if ($wildcard) {
2278         $op = 'LIKE';
2279         $value = self::getWildCardedValue($wildcard, $op, $value);
2280       }
2281
2282       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2283       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2284     }
2285     elseif ($name === 'contact_is_deleted') {
2286       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2287     }
2288     elseif (!empty($field['where'])) {
2289       $type = NULL;
2290       if (!empty($field['type'])) {
2291         $type = CRM_Utils_Type::typeToString($field['type']);
2292       }
2293
2294       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2295
2296       if (isset($locType[1]) &&
2297         is_numeric($locType[1])
2298       ) {
2299         $setTables = FALSE;
2300
2301         //get the location name
2302         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2303         $fieldName = "`$tName`.$fldName";
2304
2305         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2306         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2307
2308       }
2309       else {
2310         if ($tableName == 'civicrm_contact') {
2311           $fieldName = "contact_a.{$fieldName}";
2312         }
2313         else {
2314           $fieldName = $field['where'];
2315         }
2316       }
2317
2318       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2319       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2320         1 => $field['title'],
2321         2 => $qillop,
2322         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2323       ]);
2324
2325       if (is_array($value)) {
2326         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2327         // for api style operators for functions that hit this point without worrying about regression
2328         // (the previous comments indicated the condition for hitting this point were unknown
2329         // per CRM-14743 we are adding modified_date & created_date operator support
2330         $operations = array_keys($value);
2331         foreach ($operations as $operator) {
2332           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2333             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2334             //so this condition will satisfy the search for now
2335             if (strpos($op, 'IN') !== FALSE) {
2336               $value = [$op => $value];
2337             }
2338             // we don't know when this might happen
2339             else {
2340               throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $operator]));
2341             }
2342           }
2343         }
2344         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2345       }
2346       else {
2347         if ($wildcard) {
2348           $op = 'LIKE';
2349           $value = self::getWildCardedValue($wildcard, $op, $value);
2350         }
2351
2352         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2353       }
2354     }
2355
2356     if ($setTables && isset($field['where'])) {
2357       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2358       if (isset($tableName)) {
2359         $this->_tables[$tableName] = 1;
2360         $this->_whereTables[$tableName] = 1;
2361       }
2362     }
2363   }
2364
2365   /**
2366    * @param $where
2367    * @param $locType
2368    *
2369    * @return array
2370    * @throws Exception
2371    */
2372   public static function getLocationTableName(&$where, &$locType) {
2373     if (isset($locType[1]) && is_numeric($locType[1])) {
2374       list($tbName, $fldName) = explode(".", $where);
2375
2376       //get the location name
2377       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2378       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2379       if (in_array($locType[0], $specialFields)) {
2380         //hack to fix / special handing for phone_ext
2381         if ($locType[0] == 'phone_ext') {
2382           $locType[0] = 'phone';
2383         }
2384         if (isset($locType[2]) && $locType[2]) {
2385           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2386         }
2387         else {
2388           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2389         }
2390       }
2391       elseif (in_array($locType[0],
2392         [
2393           'address_name',
2394           'street_address',
2395           'street_name',
2396           'street_number_suffix',
2397           'street_unit',
2398           'supplemental_address_1',
2399           'supplemental_address_2',
2400           'supplemental_address_3',
2401           'city',
2402           'postal_code',
2403           'postal_code_suffix',
2404           'geo_code_1',
2405           'geo_code_2',
2406           'master_id',
2407         ]
2408       )) {
2409         //fix for search by profile with address fields.
2410         $tName = "{$locationType[$locType[1]]}-address";
2411       }
2412       elseif (in_array($locType[0],
2413           [
2414             'on_hold',
2415             'signature_html',
2416             'signature_text',
2417             'is_bulkmail',
2418           ]
2419         )) {
2420         $tName = "{$locationType[$locType[1]]}-email";
2421       }
2422       elseif ($locType[0] == 'provider_id') {
2423         $tName = "{$locationType[$locType[1]]}-im";
2424       }
2425       elseif ($locType[0] == 'openid') {
2426         $tName = "{$locationType[$locType[1]]}-openid";
2427       }
2428       else {
2429         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2430       }
2431       $tName = str_replace(' ', '_', $tName);
2432       return [$tName, $fldName];
2433     }
2434     throw new CRM_Core_Exception('Cannot determine location table information');
2435   }
2436
2437   /**
2438    * Given a result dao, extract the values and return that array
2439    *
2440    * @param CRM_Core_DAO $dao
2441    *
2442    * @return array
2443    *   values for this query
2444    */
2445   public function store($dao) {
2446     $value = [];
2447
2448     foreach ($this->_element as $key => $dontCare) {
2449       if (property_exists($dao, $key)) {
2450         if (strpos($key, '-') !== FALSE) {
2451           $values = explode('-', $key);
2452           $lastElement = array_pop($values);
2453           $current = &$value;
2454           $cnt = count($values);
2455           $count = 1;
2456           foreach ($values as $v) {
2457             if (!array_key_exists($v, $current)) {
2458               $current[$v] = [];
2459             }
2460             //bad hack for im_provider
2461             if ($lastElement == 'provider_id') {
2462               if ($count < $cnt) {
2463                 $current = &$current[$v];
2464               }
2465               else {
2466                 $lastElement = "{$v}_{$lastElement}";
2467               }
2468             }
2469             else {
2470               $current = &$current[$v];
2471             }
2472             $count++;
2473           }
2474
2475           $current[$lastElement] = $dao->$key;
2476         }
2477         else {
2478           $value[$key] = $dao->$key;
2479         }
2480       }
2481     }
2482     return $value;
2483   }
2484
2485   /**
2486    * Getter for tables array.
2487    *
2488    * @return array
2489    */
2490   public function tables() {
2491     return $this->_tables;
2492   }
2493
2494   /**
2495    * Sometimes used to create the from clause, but, not reliably, set
2496    * this AND set tables.
2497    *
2498    * It's unclear the intent - there is a 'simpleFrom' clause which
2499    * takes whereTables into account & a fromClause which doesn't.
2500    *
2501    * logic may have eroded?
2502    *
2503    * @return array
2504    */
2505   public function whereTables() {
2506     return $this->_whereTables;
2507   }
2508
2509   /**
2510    * Generate the where clause (used in match contacts and permissions)
2511    *
2512    * @param array $params
2513    * @param array $fields
2514    * @param array $tables
2515    * @param $whereTables
2516    * @param bool $strict
2517    *
2518    * @return string
2519    * @throws \CRM_Core_Exception
2520    */
2521   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2522     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2523       FALSE, $strict
2524     );
2525
2526     $tables = array_merge($query->tables(), $tables);
2527     $whereTables = array_merge($query->whereTables(), $whereTables);
2528
2529     return $query->_whereClause;
2530   }
2531
2532   /**
2533    * Create the from clause.
2534    *
2535    * @param array $tables
2536    *   Tables that need to be included in this from clause. If null,
2537    *   return mimimal from clause (i.e. civicrm_contact).
2538    * @param array $inner
2539    *   Tables that should be inner-joined.
2540    * @param array $right
2541    *   Tables that should be right-joined.
2542    * @param bool $primaryLocation
2543    *   Search on primary location. See note below.
2544    * @param int $mode
2545    *   Determines search mode based on bitwise MODE_* constants.
2546    * @param string|NULL $apiEntity
2547    *   Determines search mode based on entity by string.
2548    *
2549    * The $primaryLocation flag only seems to be used when
2550    * locationType() has been called. This may be a search option
2551    * exposed, or perhaps it's a "search all details" approach which
2552    * predates decoupling of location types and primary fields?
2553    *
2554    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2555    *
2556    * @return string
2557    *   the from clause
2558    */
2559   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2560
2561     $from = ' FROM civicrm_contact contact_a';
2562     if (empty($tables)) {
2563       return $from;
2564     }
2565
2566     if (!empty($tables['civicrm_worldregion'])) {
2567       $tables = array_merge(['civicrm_country' => 1], $tables);
2568     }
2569
2570     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2571         !empty($tables['civicrm_county'])) && empty($tables['civicrm_address'])) {
2572       $tables = array_merge(['civicrm_address' => 1],
2573         $tables
2574       );
2575     }
2576
2577     // add group_contact and group table is subscription history is present
2578     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2579       $tables = array_merge([
2580         'civicrm_group' => 1,
2581         'civicrm_group_contact' => 1,
2582       ],
2583         $tables
2584       );
2585     }
2586
2587     // to handle table dependencies of components
2588     CRM_Core_Component::tableNames($tables);
2589     // to handle table dependencies of hook injected tables
2590     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2591
2592     //format the table list according to the weight
2593     $info = CRM_Core_TableHierarchy::info();
2594
2595     foreach ($tables as $key => $value) {
2596       $k = 99;
2597       if (strpos($key, '-') !== FALSE) {
2598         $keyArray = explode('-', $key);
2599         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2600       }
2601       elseif (strpos($key, '_') !== FALSE) {
2602         $keyArray = explode('_', $key);
2603         if (is_numeric(array_pop($keyArray))) {
2604           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2605         }
2606         else {
2607           $k = CRM_Utils_Array::value($key, $info, 99);
2608         }
2609       }
2610       else {
2611         $k = CRM_Utils_Array::value($key, $info, 99);
2612       }
2613       $tempTable[$k . ".$key"] = $key;
2614     }
2615     ksort($tempTable);
2616     $newTables = [];
2617     foreach ($tempTable as $key) {
2618       $newTables[$key] = $tables[$key];
2619     }
2620
2621     $tables = $newTables;
2622
2623     foreach ($tables as $name => $value) {
2624       if (!$value) {
2625         continue;
2626       }
2627
2628       if (!empty($inner[$name])) {
2629         $side = 'INNER';
2630       }
2631       elseif (!empty($right[$name])) {
2632         $side = 'RIGHT';
2633       }
2634       else {
2635         $side = 'LEFT';
2636       }
2637
2638       if ($value != 1) {
2639         // if there is already a join statement in value, use value itself
2640         if (strpos($value, 'JOIN')) {
2641           $from .= " $value ";
2642         }
2643         else {
2644           $from .= " $side JOIN $name ON ( $value ) ";
2645         }
2646         continue;
2647       }
2648
2649       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2650     }
2651     return $from;
2652   }
2653
2654   /**
2655    * Get join statements for the from clause depending on entity type
2656    *
2657    * @param string $name
2658    * @param int $mode
2659    * @param string $side
2660    * @param string $primaryLocation
2661    * @return string
2662    */
2663   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2664     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2665     switch ($name) {
2666       case 'civicrm_address':
2667         //CRM-14263 further handling of address joins further down...
2668         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2669
2670       case 'civicrm_state_province':
2671         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2672         // but that will have been implied-in by the calling function.
2673         // test cover in testContactIDQuery
2674         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2675
2676       case 'civicrm_country':
2677         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2678         // but that will have been implied-in by the calling function.
2679         // test cover in testContactIDQuery
2680         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2681
2682       case 'civicrm_phone':
2683         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2684
2685       case 'civicrm_email':
2686         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2687
2688       case 'civicrm_im':
2689         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2690
2691       case 'im_provider':
2692         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2693         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2694         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2695         return $from;
2696
2697       case 'civicrm_openid':
2698         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2699
2700       case 'civicrm_worldregion':
2701         // We can be sure from the calling function that country will already be joined in.
2702         // we really don't need world_region - we could use a pseudoconstant for it.
2703         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2704
2705       case 'civicrm_location_type':
2706         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2707
2708       case 'civicrm_group':
2709         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2710
2711       case 'civicrm_group_contact':
2712         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2713
2714       case 'civicrm_group_contact_cache':
2715         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2716
2717       case 'civicrm_activity':
2718       case 'civicrm_activity_tag':
2719       case 'activity_type':
2720       case 'activity_status':
2721       case 'parent_id':
2722       case 'civicrm_activity_contact':
2723       case 'source_contact':
2724       case 'activity_priority':
2725         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2726
2727       case 'civicrm_entity_tag':
2728         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2729         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2730
2731       case 'civicrm_note':
2732         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2733         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2734
2735       case 'civicrm_subscription_history':
2736         $from = " $side JOIN civicrm_subscription_history";
2737         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2738         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2739
2740       case 'civicrm_relationship':
2741         if (self::$_relType == 'reciprocal') {
2742           if (self::$_relationshipTempTable) {
2743             // we have a temptable to join on
2744             $tbl = self::$_relationshipTempTable;
2745             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2746           }
2747           else {
2748             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2749             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2750             return $from;
2751           }
2752         }
2753         elseif (self::$_relType == 'b') {
2754           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2755           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2756         }
2757         else {
2758           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2759           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2760         }
2761
2762       case 'civicrm_log':
2763         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2764         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2765
2766       case 'civicrm_tag':
2767         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2768
2769       case 'civicrm_grant':
2770         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2771
2772       case 'civicrm_website':
2773         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2774
2775       case 'civicrm_campaign':
2776         //Move to default case if not in either mode.
2777         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2778           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2779         }
2780         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2781           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2782         }
2783         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2784           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2785         }
2786
2787       default:
2788         $locationTypeName = '';
2789         if (strpos($name, '-address') != 0) {
2790           $locationTypeName = 'address';
2791         }
2792         elseif (strpos($name, '-phone') != 0) {
2793           $locationTypeName = 'phone';
2794         }
2795         elseif (strpos($name, '-email') != 0) {
2796           $locationTypeName = 'email';
2797         }
2798         elseif (strpos($name, '-im') != 0) {
2799           $locationTypeName = 'im';
2800         }
2801         elseif (strpos($name, '-openid') != 0) {
2802           $locationTypeName = 'openid';
2803         }
2804
2805         if ($locationTypeName) {
2806           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2807           $parts = explode('-', $name);
2808           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2809           foreach ($locationTypes as $locationTypeID => $locationType) {
2810             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2811               $locationID = $locationTypeID;
2812             }
2813           }
2814           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2815         }
2816         else {
2817           $from = CRM_Core_Component::from($name, $mode, $side);
2818         }
2819         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2820
2821         return $from;
2822     }
2823   }
2824
2825   /**
2826    * WHERE / QILL clause for deleted_contacts
2827    *
2828    * @param array $values
2829    */
2830   public function deletedContacts($values) {
2831     list($_, $_, $value, $grouping, $_) = $values;
2832     if ($value) {
2833       // *prepend* to the relevant grouping as this is quite an important factor
2834       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2835     }
2836   }
2837
2838   /**
2839    * Where / qill clause for contact_type
2840    *
2841    * @param $values
2842    *
2843    * @throws \CRM_Core_Exception
2844    */
2845   public function contactType(&$values) {
2846     list($name, $op, $value, $grouping, $wildcard) = $values;
2847
2848     $subTypes = [];
2849     $clause = [];
2850
2851     // account for search builder mapping multiple values
2852     if (!is_array($value)) {
2853       $values = self::parseSearchBuilderString($value, 'String');
2854       if (is_array($values)) {
2855         $value = array_flip($values);
2856       }
2857     }
2858
2859     if (is_array($value)) {
2860       foreach ($value as $k => $v) {
2861         // fix for CRM-771
2862         if ($k) {
2863           $subType = NULL;
2864           $contactType = $k;
2865           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2866             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2867           }
2868
2869           if (!empty($subType)) {
2870             $subTypes[$subType] = 1;
2871           }
2872           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2873         }
2874       }
2875     }
2876     else {
2877       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2878       $contactType = $contactTypeANDSubType[0];
2879       $subType = $contactTypeANDSubType[1] ?? NULL;
2880       if (!empty($subType)) {
2881         $subTypes[$subType] = 1;
2882       }
2883       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2884     }
2885
2886     // fix for CRM-771
2887     if (!empty($clause)) {
2888       $quill = $clause;
2889       if ($op == 'IN' || $op == 'NOT IN') {
2890         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2891       }
2892       else {
2893         $type = array_pop($clause);
2894         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2895       }
2896
2897       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2898
2899       if (!empty($subTypes)) {
2900         $this->includeContactSubTypes($subTypes, $grouping);
2901       }
2902     }
2903   }
2904
2905   /**
2906    * Where / qill clause for contact_sub_type.
2907    *
2908    * @param array $values
2909    */
2910   public function contactSubType(&$values) {
2911     list($name, $op, $value, $grouping, $wildcard) = $values;
2912     $this->includeContactSubTypes($value, $grouping, $op);
2913   }
2914
2915   /**
2916    * @param $value
2917    * @param $grouping
2918    * @param string $op
2919    *
2920    * @throws \CRM_Core_Exception
2921    */
2922   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2923
2924     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2925       $op = key($value);
2926       $value = $value[$op];
2927     }
2928
2929     $clause = [];
2930     $alias = "contact_a.contact_sub_type";
2931     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2932
2933     $op = str_replace('IN', 'LIKE', $op);
2934     $op = str_replace('=', 'LIKE', $op);
2935     $op = str_replace('!', 'NOT ', $op);
2936
2937     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2938       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2939     }
2940     elseif (is_array($value)) {
2941       foreach ($value as $k => $v) {
2942         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2943       }
2944     }
2945     else {
2946       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
2947     }
2948
2949     if (!empty($clause)) {
2950       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
2951     }
2952     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
2953   }
2954
2955   /**
2956    * Where / qill clause for groups.
2957    *
2958    * @param $values
2959    *
2960    * @throws \CRM_Core_Exception
2961    * @throws \Exception
2962    */
2963   public function group($values) {
2964     list($name, $op, $value, $grouping, $wildcard) = $values;
2965
2966     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
2967     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2968       $op = key($value);
2969       $value = $value[$op];
2970     }
2971     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
2972     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
2973     // even one group might equate to multiple when looking at children so IN is simpler.
2974     // @todo - also look at != casting but there are rows below to review.
2975     $opReplacements = [
2976       'IS EMPTY' => 'IS NULL',
2977       'IS NOT EMPTY' => 'IS NOT NULL',
2978       '=' => 'IN',
2979     ];
2980     if (isset($opReplacements[$op])) {
2981       $op = $opReplacements[$op];
2982     }
2983
2984     if (strpos($op, 'NULL')) {
2985       $value = NULL;
2986     }
2987
2988     if (is_array($value) && count($value) > 1) {
2989       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
2990         throw new CRM_Core_Exception(ts("%1 is not a valid operator", [1 => $op]));
2991       }
2992       $this->_useDistinct = TRUE;
2993     }
2994
2995     if (isset($value)) {
2996       $value = CRM_Utils_Array::value($op, $value, $value);
2997     }
2998
2999     if ($name === 'group_type') {
3000       $value = array_keys($this->getGroupsFromTypeCriteria($value));
3001     }
3002
3003     $regularGroupIDs = $smartGroupIDs = [];
3004     foreach ((array) $value as $id) {
3005       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3006         $smartGroupIDs[] = (int) $id;
3007       }
3008       else {
3009         $regularGroupIDs[] = (int) trim($id);
3010       }
3011     }
3012     $hasNonSmartGroups = count($regularGroupIDs);
3013
3014     $isNotOp = ($op === 'NOT IN' || $op === '!=');
3015
3016     $statusJoinClause = $this->getGroupStatusClause($grouping);
3017     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3018     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3019     $groupClause = [];
3020     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3021       // include child groups IDs if any
3022       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3023       foreach ($childGroupIds as $key => $id) {
3024         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3025           $smartGroupIDs[] = $id;
3026           unset($childGroupIds[$key]);
3027         }
3028       }
3029       if (count($childGroupIds)) {
3030         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3031       }
3032
3033       if (empty($regularGroupIDs)) {
3034         if ($isGroupStatusSearch) {
3035           $regularGroupIDs = $smartGroupIDs;
3036         }
3037         // If it is still empty we want a filter that blocks all results.
3038         if (empty($regularGroupIDs)) {
3039           $regularGroupIDs = [0];
3040         }
3041       }
3042
3043       $gcTable = '`civicrm_group_contact-' . uniqid() . "`";
3044       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3045
3046       // @todo consider just casting != to NOT IN & handling both together.
3047       if ($op === '!=') {
3048         $groupIds = '';
3049         if (!empty($regularGroupIDs)) {
3050           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $regularGroupIDs), 'CommaSeparatedIntegers');
3051         }
3052         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3053       }
3054       else {
3055         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3056       }
3057       $groupClause[] = "( {$clause} )";
3058
3059       if ($statusJoinClause) {
3060         $joinClause[] = "{$gcTable}.$statusJoinClause";
3061       }
3062       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3063     }
3064
3065     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3066     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3067     // as that info is in the group_contact table.
3068     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3069       $this->_groupUniqueKey = uniqid();
3070       $this->_groupKeys[] = $this->_groupUniqueKey;
3071       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3072       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3073       if (!empty($groupContactCacheClause)) {
3074         if ($isNotOp) {
3075           $groupIds = CRM_Utils_Type::validate(implode(',', (array) $smartGroupIDs), 'CommaSeparatedIntegers');
3076           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3077           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3078           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3079           if (strpos($op, 'IN') !== FALSE) {
3080             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3081           }
3082           else {
3083             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3084           }
3085         }
3086         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3087       }
3088     }
3089
3090     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3091     if (!empty($groupClause)) {
3092       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3093     }
3094
3095     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3096     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3097     if (strpos($op, 'NULL') === FALSE) {
3098       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3099     }
3100   }
3101
3102   /**
3103    * @return array
3104    */
3105   public function getGroupCacheTableKeys() {
3106     return $this->_groupKeys;
3107   }
3108
3109   /**
3110    * Function translates selection of group type into a list of groups.
3111    * @param $value
3112    *
3113    * @return array
3114    */
3115   public function getGroupsFromTypeCriteria($value) {
3116     $groupIds = [];
3117     foreach ((array) $value as $groupTypeValue) {
3118       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3119       $groupIds = ($groupIds + $groupList);
3120     }
3121     return $groupIds;
3122   }
3123
3124   /**
3125    * Prime smart group cache for smart groups in the search, and join
3126    * civicrm_group_contact_cache table into the query.
3127    *
3128    * @param array $groups IDs of groups specified in search criteria.
3129    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3130    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3131    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3132    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3133    *
3134    * @return string WHERE clause component for smart group criteria.
3135    * @throws \CRM_Core_Exception
3136    */
3137   public function addGroupContactCache($groups, $tableAlias, $joinTable, $op, $joinColumn = 'id') {
3138     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3139     $groupsIds = $groups;
3140
3141     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3142     if (!empty($operator[$op]) && is_array($groups)) {
3143       $op = $operator[$op];
3144     }
3145     if (!$isNullOp && !$groups) {
3146       return NULL;
3147     }
3148     elseif (strpos($op, 'IN') !== FALSE) {
3149       $groups = [$op => $groups];
3150     }
3151     elseif (is_array($groups) && count($groups)) {
3152       $groups = ['IN' => $groups];
3153     }
3154
3155     // Find all the groups that are part of a saved search.
3156     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3157     $sql = "
3158 SELECT id, cache_date, saved_search_id, children
3159 FROM   civicrm_group
3160 WHERE  $smartGroupClause
3161   AND  ( saved_search_id != 0
3162    OR    saved_search_id IS NOT NULL
3163    OR    children IS NOT NULL )
3164 ";
3165
3166     $group = CRM_Core_DAO::executeQuery($sql);
3167
3168     while ($group->fetch()) {
3169       $this->_useDistinct = TRUE;
3170       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3171         CRM_Contact_BAO_GroupContactCache::load($group);
3172       }
3173     }
3174     if ($group->N == 0 && $op != 'NOT IN') {
3175       return NULL;
3176     }
3177
3178     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3179
3180     if ($op == 'NOT IN') {
3181       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3182     }
3183     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3184   }
3185
3186   /**
3187    * Where / qill clause for cms users
3188    *
3189    * @param $values
3190    */
3191   public function ufUser(&$values) {
3192     list($name, $op, $value, $grouping, $wildcard) = $values;
3193
3194     if ($value == 1) {
3195       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3196
3197       $this->_qill[$grouping][] = ts('CMS User');
3198     }
3199     elseif ($value == 0) {
3200       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3201
3202       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3203       $this->_qill[$grouping][] = ts('Not a CMS User');
3204     }
3205   }
3206
3207   /**
3208    * All tag search specific.
3209    *
3210    * @param array $values
3211    *
3212    * @throws \CRM_Core_Exception
3213    */
3214   public function tagSearch(&$values) {
3215     list($name, $op, $value, $grouping, $wildcard) = $values;
3216
3217     $op = "LIKE";
3218     $value = "%{$value}%";
3219     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3220
3221     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3222     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3223
3224     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3225     $tTable = "`civicrm_tag-" . uniqid() . "`";
3226
3227     if ($useAllTagTypes[2]) {
3228       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3229         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3230             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3231
3232       // search tag in cases
3233       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3234       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3235       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3236         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3237             LEFT JOIN civicrm_case
3238             ON (civicrm_case_contact.case_id = civicrm_case.id
3239                 AND civicrm_case.is_deleted = 0 )
3240             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3241             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3242       // search tag in activities
3243       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3244       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3245       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3246       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3247
3248       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3249         = " LEFT JOIN civicrm_activity_contact
3250             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3251             LEFT JOIN civicrm_activity
3252             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3253             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3254             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3255             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3256
3257       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3258       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3259     }
3260     else {
3261       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3262       $tTable = "`civicrm_tag-" . uniqid() . "`";
3263       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3264       {$etTable}.entity_table = 'civicrm_contact' )
3265                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3266
3267       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3268       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3269     }
3270   }
3271
3272   /**
3273    * Where / qill clause for tag.
3274    *
3275    * @param array $values
3276    *
3277    * @throws \CRM_Core_Exception
3278    */
3279   public function tag(&$values) {
3280     list($name, $op, $value, $grouping, $wildcard) = $values;
3281
3282     // API/Search Builder format array(operator => array(values))
3283     if (is_array($value)) {
3284       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3285         $op = key($value);
3286         $value = $value[$op];
3287       }
3288       if (count($value) > 1) {
3289         $this->_useDistinct = TRUE;
3290       }
3291     }
3292
3293     if (strpos($op, 'NULL') || strpos($op, 'EMPTY')) {
3294       $value = NULL;
3295     }
3296
3297     $tagTree = CRM_Core_BAO_Tag::getChildTags();
3298     foreach ((array) $value as $tagID) {
3299       if (!empty($tagTree[$tagID])) {
3300         // make sure value is an array here (see CORE-2502)
3301         $value = array_unique(array_merge((array) $value, $tagTree[$tagID]));
3302       }
3303     }
3304
3305     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3306
3307     // implode array, then remove all spaces
3308     $value = str_replace(' ', '', implode(',', (array) $value));
3309     if (!empty($value)) {
3310       $value = CRM_Utils_Type::validate($value, 'CommaSeparatedIntegers');
3311     }
3312
3313     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3314     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3315
3316     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3317
3318     if (!empty($useAllTagTypes[2])) {
3319       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3320         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3321
3322       // search tag in cases
3323       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3324       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3325       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3326
3327       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3328         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3329             LEFT JOIN civicrm_case
3330             ON (civicrm_case_contact.case_id = civicrm_case.id
3331                 AND civicrm_case.is_deleted = 0 )
3332             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3333       // search tag in activities
3334       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3335       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3336         = " LEFT JOIN civicrm_activity_contact
3337             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3338             LEFT JOIN civicrm_activity
3339             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3340             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3341             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3342
3343       // CRM-10338
3344       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3345         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3346       }
3347       else {
3348         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3349       }
3350     }
3351     else {
3352       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3353         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3354
3355       // CRM-10338
3356       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3357         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3358         $op = str_replace('EMPTY', 'NULL', $op);
3359         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3360       }
3361       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3362       elseif ($op == '!=') {
3363         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3364       }
3365       elseif ($op == '=' || strstr($op, 'IN')) {
3366         $op = ($op == '=') ? 'IN' : $op;
3367         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3368       }
3369     }
3370     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3371   }
3372
3373   /**
3374    * Where/qill clause for notes
3375    *
3376    * @param array $values
3377    *
3378    * @throws \CRM_Core_Exception
3379    */
3380   public function notes(&$values) {
3381     list($name, $op, $value, $grouping, $wildcard) = $values;
3382
3383     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3384     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3385     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3386
3387     $this->_useDistinct = TRUE;
3388
3389     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3390       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3391
3392     $n = trim($value);
3393     $value = CRM_Core_DAO::escapeString($n);
3394     if ($wildcard) {
3395       if (strpos($value, '%') === FALSE) {
3396         $value = "%$value%";
3397       }
3398       $op = 'LIKE';
3399     }
3400     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3401       $value = NULL;
3402     }
3403
3404     $label = NULL;
3405     $clauses = [];
3406     if ($noteOption % 2 == 0) {
3407       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3408       $label = ts('Note: Body Only');
3409     }
3410     if ($noteOption % 3 == 0) {
3411       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3412       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3413     }
3414     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3415     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3416     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3417   }
3418
3419   /**
3420    * @param string $name
3421    * @param $op
3422    * @param $grouping
3423    *
3424    * @return bool
3425    */
3426   public function nameNullOrEmptyOp($name, $op, $grouping) {
3427     switch ($op) {
3428       case 'IS NULL':
3429       case 'IS NOT NULL':
3430         $this->_where[$grouping][] = "contact_a.$name $op";
3431         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3432         return TRUE;
3433
3434       case 'IS EMPTY':
3435         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3436         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3437         return TRUE;
3438
3439       case 'IS NOT EMPTY':
3440         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3441         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3442         return TRUE;
3443
3444       default:
3445         return FALSE;
3446     }
3447   }
3448
3449   /**
3450    * Where / qill clause for sort_name
3451    *
3452    * @param array $values
3453    */
3454   public function sortName(&$values) {
3455     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3456
3457     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3458     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3459       return;
3460     }
3461
3462     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3463
3464     if (!strlen($value)) {
3465       return;
3466     }
3467
3468     $config = CRM_Core_Config::singleton();
3469
3470     $sub = [];
3471
3472     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3473     $subGlue = ' OR ';
3474
3475     $firstChar = substr($value, 0, 1);
3476     $lastChar = substr($value, -1, 1);
3477     $quotes = ["'", '"'];
3478     // If string is quoted, strip quotes and otherwise don't alter it
3479     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3480       $value = trim($value, implode('', $quotes));
3481     }
3482     // Replace spaces with wildcards for a LIKE operation
3483     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3484     // Also need to check if there is space in between sort name.
3485     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE && strpos($value, ' ') !== FALSE) {
3486       $value = str_replace(' ', '%', $value);
3487     }
3488     $value = CRM_Core_DAO::escapeString(trim($value));
3489     if (strlen($value)) {
3490       $fieldsub = [];
3491       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3492       if ($fieldName == 'sort_name') {
3493         $wc = "contact_a.sort_name";
3494       }
3495       else {
3496         $wc = "contact_a.display_name";
3497       }
3498       $fieldsub[] = " ( $wc $op $value )";
3499       if ($config->includeNickNameInName) {
3500         $wc = "contact_a.nick_name";
3501         $fieldsub[] = " ( $wc $op $value )";
3502       }
3503       if ($config->includeEmailInName) {
3504         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3505       }
3506       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3507     }
3508
3509     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3510
3511     $this->_where[$grouping][] = $sub;
3512     if ($config->includeEmailInName) {
3513       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3514       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3515     }
3516     else {
3517       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3518     }
3519   }
3520
3521   /**
3522    * Where/qill clause for greeting fields.
3523    *
3524    * @param array $values
3525    *
3526    * @throws \CRM_Core_Exception
3527    */
3528   public function greetings(&$values) {
3529     list($name, $op, $value, $grouping, $wildcard) = $values;
3530     $name .= '_display';
3531
3532     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3533     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3534     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3535   }
3536
3537   /**
3538    * Where / qill clause for email
3539    *
3540    * @param array $values
3541    * @param string $isForcePrimaryOnly
3542    *
3543    * @throws \CRM_Core_Exception
3544    */
3545   protected function email(&$values, $isForcePrimaryOnly) {
3546     list($name, $op, $value, $grouping, $wildcard) = $values;
3547     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3548
3549     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3550     // so following code is use build whereClause for contact's primart email id
3551     if (!empty($isForcePrimaryOnly)) {
3552       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3553     }
3554     // @todo - this should come from the $this->_fields array
3555     $dbName = $name === 'email_id' ? 'id' : $name;
3556
3557     if (is_array($value) || $name === 'email_id') {
3558       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, [], ts('Email'));
3559       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3560       return;
3561     }
3562
3563     // Is this ever hit now? Ideally ensure always an array & handle above.
3564     $n = trim($value);
3565     if ($n) {
3566       if (substr($n, 0, 1) == '"' &&
3567         substr($n, -1, 1) == '"'
3568       ) {
3569         $n = substr($n, 1, -1);
3570         $value = CRM_Core_DAO::escapeString($n);
3571         $op = '=';
3572       }
3573       else {
3574         $value = self::getWildCardedValue($wildcard, $op, $n);
3575       }
3576       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3577       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3578     }
3579     else {
3580       $this->_qill[$grouping][] = ts('Email') . " $op ";
3581       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3582     }
3583   }
3584
3585   /**
3586    * Where / qill clause for phone number
3587    *
3588    * @param array $values
3589    *
3590    * @throws \CRM_Core_Exception
3591    */
3592   public function phone_numeric(&$values) {
3593     list($name, $op, $value, $grouping, $wildcard) = $values;
3594     // Strip non-numeric characters; allow wildcards
3595     $number = preg_replace('/[^\d%]/', '', $value);
3596     if ($number) {
3597       if (strpos($number, '%') === FALSE) {
3598         $number = "%$number%";
3599       }
3600
3601       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3602       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3603       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3604     }
3605   }
3606
3607   /**
3608    * Where / qill clause for phone type/location
3609    *
3610    * @param array $values
3611    *
3612    * @throws \CRM_Core_Exception
3613    */
3614   public function phone_option_group($values) {
3615     list($name, $op, $value, $grouping, $wildcard) = $values;
3616     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3617     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3618     $optionName = $options[$value];
3619     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3620     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3621     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3622   }
3623
3624   /**
3625    * Where / qill clause for street_address.
3626    *
3627    * @param array $values
3628    *
3629    * @throws \CRM_Core_Exception
3630    */
3631   public function street_address(&$values) {
3632     list($name, $op, $value, $grouping) = $values;
3633
3634     if (!$op) {
3635       $op = 'LIKE';
3636     }
3637
3638     $n = trim($value);
3639
3640     if ($n) {
3641       if (strpos($value, '%') === FALSE) {
3642         // only add wild card if not there
3643         $value = "%{$value}%";
3644       }
3645       $op = 'LIKE';
3646       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3647       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3648     }
3649     else {
3650       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3651       $this->_qill[$grouping][] = ts('Street') . " $op ";
3652     }
3653
3654     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3655   }
3656
3657   /**
3658    * Where / qill clause for street_unit.
3659    *
3660    * @param array $values
3661    *
3662    * @throws \CRM_Core_Exception
3663    */
3664   public function street_number(&$values) {
3665     list($name, $op, $value, $grouping, $wildcard) = $values;
3666
3667     if (!$op) {
3668       $op = '=';
3669     }
3670
3671     $n = trim($value);
3672
3673     if (strtolower($n) == 'odd') {
3674       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3675       $this->_qill[$grouping][] = ts('Street Number is odd');
3676     }
3677     elseif (strtolower($n) == 'even') {
3678       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3679       $this->_qill[$grouping][] = ts('Street Number is even');
3680     }
3681     else {
3682       $value = $n;
3683       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3684       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3685     }
3686
3687     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3688   }
3689
3690   /**
3691    * Where / qill clause for sorting by character.
3692    *
3693    * @param array $values
3694    */
3695   public function sortByCharacter(&$values) {
3696     list($name, $op, $value, $grouping, $wildcard) = $values;
3697
3698     $name = trim($value);
3699     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3700     $this->_where[$grouping][] = $cond;
3701     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3702   }
3703
3704   /**
3705    * Where / qill clause for including contact ids.
3706    */
3707   public function includeContactIDs() {
3708     if (!$this->_includeContactIds || empty($this->_params)) {
3709       return;
3710     }
3711
3712     $contactIds = [];
3713     foreach ($this->_params as $id => $values) {
3714       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3715         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3716       }
3717     }
3718     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3719     if (!empty($contactIds)) {
3720       $this->_where[0][] = ' ( contact_a.id IN (' . implode(',', $contactIds) . " ) ) ";
3721     }
3722   }
3723
3724   /**
3725    * Where / qill clause for postal code.
3726    *
3727    * @param array $values
3728    *
3729    * @throws \CRM_Core_Exception
3730    */
3731   public function postalCode(&$values) {
3732     // skip if the fields dont have anything to do with postal_code
3733     if (empty($this->_fields['postal_code'])) {
3734       return;
3735     }
3736
3737     list($name, $op, $value, $grouping, $wildcard) = $values;
3738
3739     // Handle numeric postal code range searches properly by casting the column as numeric
3740     if (is_numeric($value)) {
3741       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3742       $val = CRM_Utils_Type::escape($value, 'Integer');
3743     }
3744     else {
3745       $field = 'civicrm_address.postal_code';
3746       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3747       if (!is_array($value)) {
3748         $val = CRM_Utils_Type::escape($value, 'String');
3749       }
3750       else {
3751         // Do we need to escape values here? I would expect buildClause does.
3752         $val = $value;
3753       }
3754     }
3755
3756     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3757
3758     if ($name == 'postal_code') {
3759       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3760       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3761     }
3762     elseif ($name == 'postal_code_low') {
3763       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3764       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3765     }
3766     elseif ($name == 'postal_code_high') {
3767       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3768       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3769     }
3770   }
3771
3772   /**
3773    * Where / qill clause for location type.
3774    *
3775    * @param array $values
3776    * @param null $status
3777    *
3778    * @return string
3779    */
3780   public function locationType(&$values, $status = NULL) {
3781     list($name, $op, $value, $grouping, $wildcard) = $values;
3782
3783     if (is_array($value)) {
3784       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3785       $this->_tables['civicrm_address'] = 1;
3786       $this->_whereTables['civicrm_address'] = 1;
3787
3788       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3789       $names = [];
3790       foreach ($value as $id) {
3791         $names[] = $locationType[$id];
3792       }
3793
3794       $this->_primaryLocation = FALSE;
3795
3796       if (!$status) {
3797         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3798       }
3799       else {
3800         return implode(' ' . ts('or') . ' ', $names);
3801       }
3802     }
3803   }
3804
3805   /**
3806    * @param $values
3807    * @param bool $fromStateProvince
3808    *
3809    * @return array|NULL
3810    * @throws \CRM_Core_Exception
3811    */
3812   public function country(&$values, $fromStateProvince = TRUE) {
3813     list($name, $op, $value, $grouping, $wildcard) = $values;
3814
3815     if (!$fromStateProvince) {
3816       $stateValues = $this->getWhereValues('state_province', $grouping);
3817       if (!empty($stateValues)) {
3818         // return back to caller if there are state province values
3819         // since that handles this case
3820         return NULL;
3821       }
3822     }
3823
3824     $countryClause = $countryQill = NULL;
3825     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3826       $this->_tables['civicrm_address'] = 1;
3827       $this->_whereTables['civicrm_address'] = 1;
3828
3829       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3830       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3831       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3832
3833       if (!$fromStateProvince) {
3834         $this->_where[$grouping][] = $countryClause;
3835         $this->_qill[$grouping][] = $countryQill;
3836       }
3837     }
3838
3839     if ($fromStateProvince) {
3840       if (!empty($countryClause)) {
3841         return [
3842           $countryClause,
3843           " ...AND... " . $countryQill,
3844         ];
3845       }
3846       else {
3847         return [NULL, NULL];
3848       }
3849     }
3850   }
3851
3852   /**
3853    * Where / qill clause for county (if present).
3854    *
3855    * @param array $values
3856    * @param null $status
3857    *
3858    * @return string
3859    */
3860   public function county(&$values, $status = NULL) {
3861     list($name, $op, $value, $grouping, $wildcard) = $values;
3862
3863     if (!is_array($value)) {
3864       // force the county to be an array
3865       $value = [$value];
3866     }
3867
3868     // check if the values are ids OR names of the counties
3869     $inputFormat = 'id';
3870     foreach ($value as $v) {
3871       if (!is_numeric($v)) {
3872         $inputFormat = 'name';
3873         break;
3874       }
3875     }
3876     $names = [];
3877     if ($op == '=') {
3878       $op = 'IN';
3879     }
3880     elseif ($op == '!=') {
3881       $op = 'NOT IN';
3882     }
3883     else {
3884       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3885       $op = str_replace('EMPTY', 'NULL', $op);
3886     }
3887
3888     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3889       $clause = "civicrm_address.county_id $op";
3890     }
3891     elseif ($inputFormat == 'id') {
3892       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3893
3894       $county = CRM_Core_PseudoConstant::county();
3895       foreach ($value as $id) {
3896         $names[] = $county[$id] ?? NULL;
3897       }
3898     }
3899     else {
3900       $inputClause = [];
3901       $county = CRM_Core_PseudoConstant::county();
3902       foreach ($value as $name) {
3903         $name = trim($name);
3904         $inputClause[] = CRM_Utils_Array::key($name, $county);
3905       }
3906       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3907       $names = $value;
3908     }
3909     $this->_tables['civicrm_address'] = 1;
3910     $this->_whereTables['civicrm_address'] = 1;
3911
3912     $this->_where[$grouping][] = $clause;
3913     if (!$status) {
3914       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3915     }
3916     else {
3917       return implode(' ' . ts('or') . ' ', $names);
3918     }
3919   }
3920
3921   /**
3922    * Where / qill clause for state/province AND country (if present).
3923    *
3924    * @param array $values
3925    * @param null $status
3926    *
3927    * @return string
3928    * @throws \CRM_Core_Exception
3929    */
3930   public function stateProvince(&$values, $status = NULL) {
3931     list($name, $op, $value, $grouping, $wildcard) = $values;
3932
3933     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3934     $this->_tables['civicrm_address'] = 1;
3935     $this->_whereTables['civicrm_address'] = 1;
3936
3937     $countryValues = $this->getWhereValues('country', $grouping);
3938     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3939     if ($countryClause) {
3940       $clause = "( $stateClause AND $countryClause )";
3941     }
3942     else {
3943       $clause = $stateClause;
3944     }
3945
3946     $this->_where[$grouping][] = $clause;
3947     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3948     if (!$status) {
3949       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3950     }
3951     else {
3952       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3953     }
3954   }
3955
3956   /**
3957    * Where / qill clause for change log.
3958    *
3959    * @param array $values
3960    */
3961   public function changeLog(&$values) {
3962     list($name, $op, $value, $grouping, $wildcard) = $values;
3963
3964     $targetName = $this->getWhereValues('changed_by', $grouping);
3965     if (!$targetName) {
3966       return;
3967     }
3968
3969     $name = trim($targetName[2]);
3970     $name = CRM_Core_DAO::escapeString($name);
3971     $name = $targetName[4] ? "%$name%" : $name;
3972     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3973     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
3974     $fieldTitle = ts('Altered By');
3975
3976     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
3977     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
3978       1 => $fieldTitle,
3979       2 => $qillop,
3980       3 => $qillVal,
3981     ]);
3982   }
3983
3984   /**
3985    * @param $values
3986    *
3987    * @throws \CRM_Core_Exception
3988    */
3989   public function modifiedDates($values) {
3990     $this->_useDistinct = TRUE;
3991     CRM_Core_Error::deprecatedWarning('function should not be reachable');
3992     // CRM-11281, default to added date if not set
3993     $fieldTitle = ts('Added Date');
3994     $fieldName = 'created_date';
3995     foreach (array_keys($this->_params) as $id) {
3996       if ($this->_params[$id][0] == 'log_date') {
3997         if ($this->_params[$id][2] == 2) {
3998           $fieldTitle = ts('Modified Date');
3999           $fieldName = 'modified_date';
4000         }
4001       }
4002     }
4003
4004     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
4005
4006     self::$_openedPanes[ts('Change Log')] = TRUE;
4007   }
4008
4009   /**
4010    * @param $values
4011    *
4012    * @throws \CRM_Core_Exception
4013    */
4014   public function demographics(&$values) {
4015     list($name, $op, $value, $grouping, $wildcard) = $values;
4016
4017     if (($name == 'age_low') || ($name == 'age_high')) {
4018       $this->ageRangeQueryBuilder($values,
4019         'contact_a', 'age', 'birth_date', ts('Age')
4020       );
4021     }
4022     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4023
4024       $this->dateQueryBuilder($values,
4025         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4026       );
4027     }
4028     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4029
4030       $this->dateQueryBuilder($values,
4031         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4032       );
4033     }
4034
4035     self::$_openedPanes[ts('Demographics')] = TRUE;
4036   }
4037
4038   /**
4039    * @param $values
4040    */
4041   public function privacy(&$values) {
4042     list($name, $op, $value, $grouping) = $values;
4043     if (is_array($value)) {
4044       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4045         $op = key($value);
4046         $value = $value[$op];
4047       }
4048     }
4049     $field = $this->_fields[$name] ?? NULL;
4050     CRM_Utils_Type::validate($value, 'Integer');
4051     $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4052     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4053     $title = $field ? $field['title'] : $name;
4054     $this->_qill[$grouping][] = "$title $op $value";
4055   }
4056
4057   /**
4058    * @param $values
4059    */
4060   public function privacyOptions($values) {
4061     list($name, $op, $value, $grouping, $wildcard) = $values;
4062
4063     if (empty($value) || !is_array($value)) {
4064       return;
4065     }
4066
4067     // get the operator and toggle values
4068     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4069     $operator = 'OR';
4070     if ($opValues &&
4071       strtolower($opValues[2] == 'AND')
4072     ) {
4073       // @todo this line is logially unreachable
4074       $operator = 'AND';
4075     }
4076
4077     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4078     $compareOP = '!';
4079     if ($toggleValues &&
4080       $toggleValues[2] == 2
4081     ) {
4082       $compareOP = '';
4083     }
4084
4085     $clauses = [];
4086     $qill = [];
4087     foreach ($value as $dontCare => $pOption) {
4088       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4089       $field = $this->_fields[$pOption] ?? NULL;
4090       $title = $field ? $field['title'] : $pOption;
4091       $qill[] = " $title = 1 ";
4092     }
4093
4094     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4095     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4096   }
4097
4098   /**
4099    * @param $values
4100    *
4101    * @throws \CRM_Core_Exception
4102    */
4103   public function preferredCommunication(&$values) {
4104     list($name, $op, $value, $grouping, $wildcard) = $values;
4105
4106     if (!is_array($value)) {
4107       $value = str_replace(['(', ')'], '', explode(",", $value));
4108     }
4109     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4110       $op = key($value);
4111       $value = $value[$op];
4112     }
4113     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4114
4115     if (self::caseImportant($op)) {
4116       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4117       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4118       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4119     }
4120
4121     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4122     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4123   }
4124
4125   /**
4126    * Where / qill clause for relationship.
4127    *
4128    * @param array $values
4129    */
4130   public function relationship(&$values) {
4131     list($name, $op, $value, $grouping, $wildcard) = $values;
4132     if ($this->_relationshipValuesAdded) {
4133       return;
4134     }
4135     // also get values array for relation_target_name
4136     // for relationship search we always do wildcard
4137     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4138     $description = $this->getWhereValues('relation_description', $grouping);
4139     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4140     $relStatus = $this->getWhereValues('relation_status', $grouping);
4141     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4142
4143     $nameClause = $name = NULL;
4144     if ($targetName) {
4145       $name = trim($targetName[2]);
4146       if (substr($name, 0, 1) == '"' &&
4147         substr($name, -1, 1) == '"'
4148       ) {
4149         $name = substr($name, 1, -1);
4150         $name = CRM_Core_DAO::escapeString($name);
4151         $nameClause = "= '$name'";
4152       }
4153       else {
4154         $name = CRM_Core_DAO::escapeString($name);
4155         $nameClause = "LIKE '%{$name}%'";
4156       }
4157     }
4158
4159     $relTypes = $relTypesIds = [];
4160     if (!empty($relationType)) {
4161       $relationType[2] = (array) $relationType[2];
4162       foreach ($relationType[2] as $relType) {
4163         $rel = explode('_', $relType);
4164         self::$_relType = $rel[1];
4165         $params = ['id' => $rel[0]];
4166         $typeValues = [];
4167         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4168         if (!empty($rTypeValue)) {
4169           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4170             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4171             self::$_relType = 'reciprocal';
4172           }
4173           $relTypesIds[] = $rel[0];
4174           $relTypes[] = $relType;
4175         }
4176       }
4177     }
4178
4179     // if we are creating a temp table we build our own where for the relationship table
4180     $relationshipTempTable = NULL;
4181     if (self::$_relType == 'reciprocal') {
4182       $where = [];
4183       self::$_relationshipTempTable = $relationshipTempTable = CRM_Utils_SQL_TempTable::build()
4184         ->createWithColumns("`contact_id` int(10) unsigned NOT NULL DEFAULT '0', `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0', id int unsigned, KEY `contact_id` (`contact_id`), KEY `contact_id_alt` (`contact_id_alt`)")
4185         ->getName();
4186       if ($nameClause) {
4187         $where[$grouping][] = " sort_name $nameClause ";
4188       }
4189       $groupJoinTable = "civicrm_relationship";
4190       $groupJoinColumn = "contact_id_alt";
4191     }
4192     else {
4193       $where = &$this->_where;
4194       if ($nameClause) {
4195         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4196       }
4197       $groupJoinTable = "contact_b";
4198       $groupJoinColumn = "id";
4199     }
4200     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE, 'label', FALSE);
4201     if ($nameClause || !$targetGroup) {
4202       if (!empty($relationType)) {
4203         $relQill = '';
4204         foreach ($relTypes as $rel) {
4205           if (!empty($relQill)) {
4206             $relQill .= ' OR ';
4207           }
4208           $relQill .= $allRelationshipType[$rel];
4209         }
4210         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4211       }
4212       elseif ($name) {
4213         $this->_qill[$grouping][] = $name;
4214       }
4215     }
4216
4217     //check to see if the target contact is in specified group
4218     if ($targetGroup) {
4219       //add contacts from static groups
4220       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4221         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4222       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4223         implode(",", $targetGroup[2]) . ") ) ";
4224
4225       //add contacts from saved searches
4226       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4227
4228       //set the group where clause
4229       if ($ssWhere) {
4230         $groupWhere[] = "( " . $ssWhere . " )";
4231       }
4232       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4233
4234       //Get the names of the target groups for the qill
4235       $groupNames = CRM_Core_PseudoConstant::group();
4236       $qillNames = [];
4237       foreach ($targetGroup[2] as $groupId) {
4238         if (array_key_exists($groupId, $groupNames)) {
4239           $qillNames[] = $groupNames[$groupId];
4240         }
4241       }
4242       if (!empty($relationType)) {
4243         $relQill = '';
4244         foreach ($relTypes as $rel) {
4245           if (!empty($relQill)) {
4246             $relQill .= ' OR ';
4247           }
4248           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4249         }
4250         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4251       }
4252       else {
4253         $this->_qill[$grouping][] = implode(", ", $qillNames);
4254       }
4255     }
4256
4257     // Description
4258     if (!empty($description[2]) && trim($description[2])) {
4259       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4260       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4261       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4262     }
4263
4264     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4265     $today = date('Ymd');
4266     //check for active, inactive and all relation status
4267     if (empty($relStatus[2])) {
4268       $where[$grouping][] = "(
4269 civicrm_relationship.is_active = 1 AND
4270 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4271 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4272 )";
4273       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4274     }
4275     elseif ($relStatus[2] == 1) {
4276       $where[$grouping][] = "(
4277 civicrm_relationship.is_active = 0 OR
4278 civicrm_relationship.end_date < {$today} OR
4279 civicrm_relationship.start_date > {$today}
4280 )";
4281       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4282     }
4283
4284     $onlyDeleted = 0;
4285     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4286       $onlyDeleted = 1;
4287     }
4288     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4289
4290     $this->addRelationshipPermissionClauses($grouping, $where);
4291     $this->addRelationshipDateClauses($grouping, $where);
4292     $this->addRelationshipActivePeriodClauses($grouping, $where);
4293     if (!empty($relTypes)) {
4294       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4295     }
4296     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4297     $this->_useDistinct = TRUE;
4298     $this->_relationshipValuesAdded = TRUE;
4299     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4300     // join on that,
4301     if ($relationshipTempTable) {
4302       $whereClause = '';
4303       if (!empty($where[$grouping])) {
4304         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4305         $whereClause = str_replace('contact_b', 'c', $whereClause);
4306       }
4307       $sql = "
4308         INSERT INTO {$relationshipTempTable} (contact_id, contact_id_alt, id)
4309           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4310             FROM civicrm_relationship
4311             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4312             $whereClause )
4313           UNION
4314             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4315             FROM civicrm_relationship
4316             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4317             $whereClause )
4318       ";
4319       CRM_Core_DAO::executeQuery($sql);
4320     }
4321   }
4322
4323   /**
4324    * Add relationship permission criteria to where clause.
4325    *
4326    * @param string $grouping
4327    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4328    *   Not the main query.
4329    */
4330   public function addRelationshipPermissionClauses($grouping, &$where) {
4331     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4332     if ($relPermission) {
4333       if (!is_array($relPermission[2])) {
4334         // this form value was scalar in previous versions of Civi
4335         $relPermission[2] = [$relPermission[2]];
4336       }
4337       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4338
4339       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4340
4341       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4342       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4343     }
4344   }
4345
4346   /**
4347    * Add start & end date criteria in
4348    * @param string $grouping
4349    * @param array $where
4350    *   = array to add where clauses to, in case you are generating a temp table.
4351    * not the main query.
4352    */
4353   public function addRelationshipDateClauses($grouping, &$where) {
4354     foreach (['start_date', 'end_date'] as $dateField) {
4355       $dateValueLow = $this->getWhereValues('relationship_' . $dateField . '_low', $grouping);
4356       $dateValueHigh = $this->getWhereValues('relationship_' . $dateField . '_high', $grouping);
4357       if (!empty($dateValueLow)) {
4358         $date = date('Ymd', strtotime($dateValueLow[2]));
4359         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4360         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4361       }
4362       if (!empty($dateValueHigh)) {
4363         $date = date('Ymd', strtotime($dateValueHigh[2]));
4364         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4365         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4366       }
4367     }
4368   }
4369
4370   /**
4371    * Add start & end active period criteria in
4372    * @param string $grouping
4373    * @param array $where
4374    *   = array to add where clauses to, in case you are generating a temp table.
4375    * not the main query.
4376    */
4377   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4378     $dateValues = [];
4379     $dateField = 'active_period_date';
4380
4381     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4382     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4383     $dateValueLowFormated = $dateValueHighFormated = NULL;
4384     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4385       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4386       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4387       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4388     }
4389     elseif (!empty($dateValueLow)) {
4390       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4391       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4392     }
4393     elseif (!empty($dateValueHigh)) {
4394       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4395       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4396     }
4397
4398     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4399       $where[$grouping][] = $activePeriodClauses;
4400     }
4401   }
4402
4403   /**
4404    * Get start & end active period criteria
4405    *
4406    * @param $from
4407    * @param $to
4408    * @param $forceTableName
4409    *
4410    * @return string
4411    */
4412   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4413     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4414     if (!is_null($from) && !is_null($to)) {
4415       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4416                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4417                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4418                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4419                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4420                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4421     }
4422     elseif (!is_null($from)) {
4423       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4424                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4425                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4426     }
4427     elseif (!is_null($to)) {
4428       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4429                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4430                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4431     }
4432   }
4433
4434   /**
4435    * Default set of return properties.
4436    *
4437    * @param int $mode
4438    *
4439    * @return array
4440    *   derault return properties
4441    */
4442   public static function &defaultReturnProperties($mode = 1) {
4443     if (!isset(self::$_defaultReturnProperties)) {
4444       self::$_defaultReturnProperties = [];
4445     }
4446
4447     if (!isset(self::$_defaultReturnProperties[$mode])) {
4448       // add activity return properties
4449       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4450         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4451       }
4452       else {
4453         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4454       }
4455
4456       if (empty(self::$_defaultReturnProperties[$mode])) {
4457         self::$_defaultReturnProperties[$mode] = [
4458           'image_URL' => 1,
4459           'legal_identifier' => 1,
4460           'external_identifier' => 1,
4461           'contact_type' => 1,
4462           'contact_sub_type' => 1,
4463           'sort_name' => 1,
4464           'display_name' => 1,
4465           'preferred_mail_format' => 1,
4466           'nick_name' => 1,
4467           'first_name' => 1,
4468           'middle_name' => 1,
4469           'last_name' => 1,
4470           'prefix_id' => 1,
4471           'suffix_id' => 1,
4472           'formal_title' => 1,
4473           'communication_style_id' => 1,
4474           'birth_date' => 1,
4475           'gender_id' => 1,
4476           'street_address' => 1,
4477           'supplemental_address_1' => 1,
4478           'supplemental_address_2' => 1,
4479           'supplemental_address_3' => 1,
4480           'city' => 1,
4481           'postal_code' => 1,
4482           'postal_code_suffix' => 1,
4483           'state_province' => 1,
4484           'country' => 1,
4485           'world_region' => 1,
4486           'geo_code_1' => 1,
4487           'geo_code_2' => 1,
4488           'email' => 1,
4489           'on_hold' => 1,
4490           'phone' => 1,
4491           'im' => 1,
4492           'household_name' => 1,
4493           'organization_name' => 1,
4494           'deceased_date' => 1,
4495           'is_deceased' => 1,
4496           'job_title' => 1,
4497           'legal_name' => 1,
4498           'sic_code' => 1,
4499           'current_employer' => 1,
4500           // FIXME: should we use defaultHierReturnProperties() for the below?
4501           'do_not_email' => 1,
4502           'do_not_mail' => 1,
4503           'do_not_sms' => 1,
4504           'do_not_phone' => 1,
4505           'do_not_trade' => 1,
4506           'is_opt_out' => 1,
4507           'contact_is_deleted' => 1,
4508           'preferred_communication_method' => 1,
4509           'preferred_language' => 1,
4510         ];
4511       }
4512     }
4513     return self::$_defaultReturnProperties[$mode];
4514   }
4515
4516   /**
4517    * Get primary condition for a sql clause.
4518    *
4519    * @param int $value
4520    *
4521    * @return string|NULL
4522    */
4523   public static function getPrimaryCondition($value) {
4524     if (is_numeric($value)) {
4525       $value = (int ) $value;
4526       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4527     }
4528     return NULL;
4529   }
4530
4531   /**
4532    * Wrapper for a simple search query.
4533    *
4534    * @param array $params
4535    * @param array $returnProperties
4536    * @param bool $count
4537    *
4538    * @return string
4539    * @throws \CRM_Core_Exception
4540    */
4541   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4542     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4543     list($select, $from, $where, $having) = $query->query();
4544     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4545
4546     $query = "$select $from $where $groupBy $having";
4547     return $query;
4548   }
4549
4550   /**
4551    * These are stub comments as this function needs more explanation - particularly in terms of how it
4552    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4553    *
4554    * This function was originally written as a wrapper for the api query but is called from multiple places
4555    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4556    * but it is unclear as to whehter that is historical or there is a reason
4557    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4558    *
4559    * @param array $params
4560    * @param array $returnProperties
4561    * @param null $fields
4562    * @param string $sort
4563    * @param int $offset
4564    * @param int $row_count
4565    * @param bool $smartGroupCache
4566    *   ?? update smart group cache?.
4567    * @param bool $count
4568    *   Return count obnly.
4569    * @param bool $skipPermissions
4570    *   Should permissions be ignored or should the logged in user's permissions be applied.
4571    * @param int $mode
4572    *   This basically correlates to the component.
4573    * @param string $apiEntity
4574    *   The api entity being called.
4575    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4576    *
4577    * @param bool|null $primaryLocationOnly
4578    *
4579    * @return array
4580    * @throws \CRM_Core_Exception
4581    */
4582   public static function apiQuery(
4583     $params = NULL,
4584     $returnProperties = NULL,
4585     $fields = NULL,
4586     $sort = NULL,
4587     $offset = 0,
4588     $row_count = 25,
4589     $smartGroupCache = TRUE,
4590     $count = FALSE,
4591     $skipPermissions = TRUE,
4592     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4593     $apiEntity = NULL,
4594     $primaryLocationOnly = NULL
4595   ) {
4596
4597     $query = new CRM_Contact_BAO_Query(
4598       $params, $returnProperties,
4599       NULL, TRUE, FALSE, $mode,
4600       $skipPermissions,
4601       TRUE, $smartGroupCache,
4602       NULL, 'AND',
4603       $apiEntity, $primaryLocationOnly
4604     );
4605
4606     //this should add a check for view deleted if permissions are enabled
4607     if ($skipPermissions) {
4608       $query->_skipDeleteClause = TRUE;
4609     }
4610     $query->generatePermissionClause(FALSE, $count);
4611
4612     // note : this modifies _fromClause and _simpleFromClause
4613     $query->includePseudoFieldsJoin($sort);
4614
4615     list($select, $from, $where, $having) = $query->query($count);
4616
4617     if (!empty($query->_permissionWhereClause)) {
4618       if (!empty($query->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
4619         $from .= " $query->_permissionFromClause";
4620       }
4621       if (empty($where)) {
4622         $where = "WHERE $query->_permissionWhereClause";
4623       }
4624       else {
4625         $where = "$where AND $query->_permissionWhereClause";
4626       }
4627     }
4628
4629     $sql = "$select $from $where $having";
4630
4631     // add group by only when API action is not getcount
4632     //  otherwise query fetches incorrect count
4633     if ($query->_useGroupBy && !$count) {
4634       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4635     }
4636     if (!empty($sort)) {
4637       $sort = CRM_Utils_Type::escape($sort, 'String');
4638       $sql .= " ORDER BY $sort ";
4639     }
4640     if ($row_count > 0 && $offset >= 0) {
4641       $offset = CRM_Utils_Type::escape($offset, 'Int');
4642       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4643       $sql .= " LIMIT $offset, $row_count ";
4644     }
4645
4646     $dao = CRM_Core_DAO::executeQuery($sql);
4647
4648     // @todo derive this from the component class rather than hard-code two options.
4649     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4650
4651     $values = [];
4652     while ($dao->fetch()) {
4653       if ($count) {
4654         $noRows = $dao->rowCount;
4655         return [$noRows, NULL];
4656       }
4657       $val = $query->store($dao);
4658       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4659
4660       if (!empty($convertedVals)) {
4661         $val = array_replace_recursive($val, $convertedVals);
4662       }
4663       $values[$dao->$entityIDField] = $val;
4664     }
4665     return [$values];
4666   }
4667
4668   /**
4669    * Get the actual custom field name by stripping off the appended string.
4670    *
4671    * The string could be _relative, _from, or _to
4672    *
4673    * @todo use metadata rather than convention to do this.
4674    *
4675    * @param string $parameterName
4676    *   The name of the parameter submitted to the form.
4677    *   e.g
4678    *   custom_3_relative
4679    *   custom_3_from
4680    *
4681    * @return string
4682    */
4683   public static function getCustomFieldName($parameterName) {
4684     if (substr($parameterName, -5, 5) == '_from') {
4685       return substr($parameterName, 0, strpos($parameterName, '_from'));
4686     }
4687     if (substr($parameterName, -9, 9) == '_relative') {
4688       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4689     }
4690     if (substr($parameterName, -3, 3) == '_to') {
4691       return substr($parameterName, 0, strpos($parameterName, '_to'));
4692     }
4693   }
4694
4695   /**
4696    * Convert submitted values for relative custom fields to query object format.
4697    *
4698    * The query will support the sqlOperator format so convert to that format.
4699    *
4700    * @param array $formValues
4701    *   Submitted values.
4702    * @param array $params
4703    *   Converted parameters for the query object.
4704    * @param string $values
4705    *   Submitted value.
4706    * @param string $fieldName
4707    *   Submitted field name. (Matches form field not DB field.)
4708    */
4709   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4710     if (empty($values)) {
4711       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4712       // this function gets called again after they fields have been converted which can get ugly.
4713       return;
4714     }
4715     $customFieldName = self::getCustomFieldName($fieldName);
4716
4717     if (substr($fieldName, -9, 9) == '_relative') {
4718       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4719     }
4720     else {
4721       if ($fieldName == $customFieldName . '_to' && !empty($formValues[$customFieldName . '_from'])) {
4722         // Both to & from are set. We only need to acton one, choosing from.
4723         return;
4724       }
4725
4726       $from = $formValues[$customFieldName . '_from'] ?? NULL;
4727       $to = $formValues[$customFieldName . '_to'] ?? NULL;
4728
4729       if (self::isCustomDateField($customFieldName)) {
4730         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4731       }
4732     }
4733
4734     if ($from) {
4735       if ($to) {
4736         $relativeFunction = ['BETWEEN' => [$from, $to]];
4737       }
4738       else {
4739         $relativeFunction = ['>=' => $from];
4740       }
4741     }
4742     else {
4743       $relativeFunction = ['<=' => $to];
4744     }
4745     $params[] = [
4746       $customFieldName,
4747       '=',
4748       $relativeFunction,
4749       0,
4750       0,
4751     ];
4752   }
4753
4754   /**
4755    * Are we dealing with custom field of type date.
4756    *
4757    * @param $fieldName
4758    *
4759    * @return bool
4760    * @throws \CiviCRM_API3_Exception
4761    */
4762   public static function isCustomDateField($fieldName) {
4763     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4764       return FALSE;
4765     }
4766     try {
4767       $customFieldDataType = civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type']);
4768       if ('Date' == $customFieldDataType) {
4769         return TRUE;
4770       }
4771     }
4772     catch (CiviCRM_API3_Exception $e) {
4773     }
4774     return FALSE;
4775   }
4776
4777   /**
4778    * Has this field already been reformatting to Query object syntax.
4779    *
4780    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4781    * to come with associated double queries & is possibly damaging performance.
4782    *
4783    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4784    * them as they are.
4785    *
4786    * @param mixed $values
4787    *   Value in formValues for the field.
4788    *
4789    * @return bool;
4790    */
4791   public static function isAlreadyProcessedForQueryFormat($values) {
4792     if (!is_array($values)) {
4793       return FALSE;
4794     }
4795     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4796       return FALSE;
4797     }
4798     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4799   }
4800
4801   /**
4802    * If the state and country are passed remove state.
4803    *
4804    * Country is implicit from the state, but including both results in
4805    * a poor query as there is no combined index on state AND country.
4806    *
4807    * @see https://issues.civicrm.org/jira/browse/CRM-18125
4808    *
4809    * @param array $formValues
4810    */
4811   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4812     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4813       // The use of array map sanitises the data by ensuring we are dealing with integers.
4814       $states = implode(', ', array_map('intval', $formValues['state_province']));
4815       $countryList = CRM_Core_DAO::singleValueQuery(
4816         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4817       );
4818       if ($countryList == $formValues['country']) {
4819         unset($formValues['country']);
4820       }
4821     }
4822   }
4823
4824   /**
4825    * For some special cases, grouping by subset of select fields becomes mandatory.
4826    * Hence, full_group_by mode is handled by appending any_value
4827    * keyword to select fields not present in groupBy
4828    *
4829    * @param array $selectClauses
4830    * @param array $groupBy - Columns already included in GROUP By clause.
4831    * @param string $aggregateFunction
4832    *
4833    * @return string
4834    */
4835   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4836     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4837       $groupBy = array_map('trim', (array) $groupBy);
4838       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4839       foreach ($selectClauses as $key => &$val) {
4840         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4841         // append ANY_VALUE() keyword
4842         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4843           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4844             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4845             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4846         }
4847       }
4848     }
4849
4850     return "SELECT " . implode(', ', $selectClauses) . " ";
4851   }
4852
4853   /**
4854    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4855    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4856    * keyword to select fields not present in groupBy
4857    *
4858    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4859    * @param array $orderBys - ORDER BY sub-clauses
4860    *
4861    */
4862   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4863     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4864       foreach ($orderBys as $orderBy) {
4865         // remove sort syntax from ORDER BY clauses if present
4866         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4867         // if ORDER BY column is not present in GROUP BY then append it to end
4868         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4869           $groupBy .= ", {$orderBy}";
4870         }
4871       }
4872     }
4873   }
4874
4875   /**
4876    * Include Select columns in groupBy clause.
4877    *
4878    * @param array $selectClauses
4879    * @param array $groupBy - Columns already included in GROUP By clause.
4880    *
4881    * @return string
4882    */
4883   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4884     $groupBy = (array) $groupBy;
4885     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4886     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4887
4888     //return if ONLY_FULL_GROUP_BY is not enabled.
4889     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4890       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4891       foreach ($selectClauses as $key => $val) {
4892         $aliasArray = preg_split('/ as /i', $val);
4893         // if more than 1 alias we need to split by ','.
4894         if (count($aliasArray) > 2) {
4895           $aliasArray = preg_split('/,/', $val);
4896           foreach ($aliasArray as $key => $value) {
4897             $alias = current(preg_split('/ as /i', $value));
4898             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4899               $groupBy[] = $alias;
4900             }
4901           }
4902         }
4903         else {
4904           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4905           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4906           $tableName = current(explode('.', $selectColumn));
4907           $primaryKey = "{$tableName}.id";
4908           // exclude columns which are already included in groupBy and aggregate functions from select
4909           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4910           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4911             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4912               $groupBy[] = $alias;
4913             }
4914             else {
4915               $groupBy[] = $selectColumn;
4916             }
4917           }
4918         }
4919       }
4920     }
4921
4922     if (!empty($groupBy)) {
4923       return " GROUP BY " . implode(', ', $groupBy);
4924     }
4925     return '';
4926   }
4927
4928   /**
4929    * Create and query the db for an contact search.
4930    *
4931    * @param int $offset
4932    *   The offset for the query.
4933    * @param int $rowCount
4934    *   The number of rows to return.
4935    * @param string|CRM_Utils_Sort $sort
4936    *   The order by string.
4937    * @param bool $count
4938    *   Is this a count only query ?.
4939    * @param bool $includeContactIds
4940    *   Should we include contact ids?.
4941    * @param bool $sortByChar
4942    *   If true returns the distinct array of first characters for search results.
4943    * @param bool $groupContacts
4944    *   If true, return only the contact ids.
4945    * @param bool $returnQuery
4946    *   Should we return the query as a string.
4947    * @param string $additionalWhereClause
4948    *   If the caller wants to further restrict the search (used for components).
4949    * @param null $sortOrder
4950    * @param string $additionalFromClause
4951    *   Should be clause with proper joins, effective to reduce where clause load.
4952    *
4953    * @param bool $skipOrderAndLimit
4954    *
4955    * @return string|null|CRM_Core_DAO
4956    */
4957   public function searchQuery(
4958     $offset = 0, $rowCount = 0, $sort = NULL,
4959     $count = FALSE, $includeContactIds = FALSE,
4960     $sortByChar = FALSE, $groupContacts = FALSE,
4961     $returnQuery = FALSE,
4962     $additionalWhereClause = NULL, $sortOrder = NULL,
4963     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4964   ) {
4965
4966     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4967
4968     if ($returnQuery) {
4969       return $query;
4970     }
4971     if ($count) {
4972       return CRM_Core_DAO::singleValueQuery($query);
4973     }
4974
4975     $dao = CRM_Core_DAO::executeQuery($query);
4976
4977     // We can always call this - it will only re-enable if it was originally enabled.
4978     CRM_Core_DAO::reenableFullGroupByMode();
4979
4980     if ($groupContacts) {
4981       $ids = [];
4982       while ($dao->fetch()) {
4983         $ids[] = $dao->id;
4984       }
4985       return implode(',', $ids);
4986     }
4987
4988     return $dao;
4989   }
4990
4991   /**
4992    * Create and query the db for the list of all first letters used by contacts
4993    *
4994    * @return CRM_Core_DAO
4995    */
4996   public function alphabetQuery() {
4997     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
4998     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
4999       {$sqlParts['from']}
5000       {$sqlParts['where']}";
5001     $dao = CRM_Core_DAO::executeQuery($query);
5002     return $dao;
5003   }
5004
5005   /**
5006    * Fetch a list of contacts for displaying a search results page
5007    *
5008    * @param array $cids
5009    *   List of contact IDs
5010    * @param bool $includeContactIds
5011    * @return CRM_Core_DAO
5012    */
5013   public function getCachedContacts($cids, $includeContactIds) {
5014     CRM_Core_DAO::disableFullGroupByMode();
5015     CRM_Utils_Type::validateAll($cids, 'Positive');
5016     $this->_includeContactIds = $includeContactIds;
5017     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5018     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5019     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5020     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5021     $order = 'ORDER BY _wgt';
5022     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5023     $limit = '';
5024     $query = "$select $from $where $groupBy $order $limit";
5025
5026     $result = CRM_Core_DAO::executeQuery($query);
5027     CRM_Core_DAO::reenableFullGroupByMode();
5028     return $result;
5029   }
5030
5031   /**
5032    * Construct a SQL CASE expression.
5033    *
5034    * @param string $idCol
5035    *   The name of a column with ID's (eg 'contact_a.id').
5036    * @param array $cids
5037    *   Array(int $weight => int $id).
5038    * @return string
5039    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5040    */
5041   private function createSqlCase($idCol, $cids) {
5042     $buf = "CASE\n";
5043     foreach ($cids as $weight => $cid) {
5044       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5045     }
5046     $buf .= "END\n";
5047     return $buf;
5048   }
5049
5050   /**
5051    * Populate $this->_permissionWhereClause with permission related clause and update other
5052    * query related properties.
5053    *
5054    * Function calls ACL permission class and hooks to filter the query appropriately
5055    *
5056    * Note that these 2 params were in the code when extracted from another function
5057    * and a second round extraction would be to make them properties of the class
5058    *
5059    * @param bool $onlyDeleted
5060    *   Only get deleted contacts.
5061    * @param bool $count
5062    *   Return Count only.
5063    */
5064   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5065     if (!$this->_skipPermission) {
5066       $permissionClauses = CRM_Contact_BAO_Contact_Permission::cacheClause();
5067       $this->_permissionWhereClause = $permissionClauses[1];
5068       $this->_permissionFromClause = $permissionClauses[0];
5069
5070       if (CRM_Core_Permission::check('access deleted contacts')) {
5071         if (!$onlyDeleted) {
5072           $this->_permissionWhereClause .= ' AND (contact_a.is_deleted = 0)';
5073         }
5074         else {
5075           $this->_permissionWhereClause .= " AND (contact_a.is_deleted) ";
5076         }
5077       }
5078
5079       if (isset($this->_tables['civicrm_activity'])) {
5080         $bao = new CRM_Activity_BAO_Activity();
5081         $clauses = $subclauses = [];
5082         foreach ((array) $bao->addSelectWhereClause() as $field => $vals) {
5083           if ($vals && $field !== 'id') {
5084             $clauses[] = $bao->tableName() . ".$field " . $vals;
5085           }
5086           elseif ($vals) {
5087             $subclauses[] = "$field " . implode(" AND $field ", (array) $vals);
5088           }
5089         }
5090         if ($subclauses) {
5091           $clauses[] = $bao->tableName() . '.`id` IN (SELECT `id` FROM `' . $bao->tableName() . '` WHERE ' . implode(' AND ', $subclauses) . ')';
5092         }
5093         if (!empty($clauses) && $this->_permissionWhereClause) {
5094           $this->_permissionWhereClause .= ' AND (' . implode(' AND ', $clauses) . ')';
5095         }
5096         elseif (!empty($clauses)) {
5097           $this->_permissionWhereClause .= '(' . implode(' AND ', $clauses) . ')';
5098         }
5099       }
5100     }
5101     else {
5102       // add delete clause if needed even if we are skipping permission
5103       // CRM-7639
5104       if (!$this->_skipDeleteClause) {
5105         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5106           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5107         }
5108         else {
5109           // CRM-6181
5110           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5111         }
5112       }
5113     }
5114   }
5115
5116   /**
5117    * @param $val
5118    */
5119   public function setSkipPermission($val) {
5120     $this->_skipPermission = $val;
5121   }
5122
5123   /**
5124    * @param null $context
5125    *
5126    * @return array
5127    * @throws \CRM_Core_Exception
5128    */
5129   public function summaryContribution($context = NULL) {
5130     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5131     if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5132       $from .= " $this->_permissionFromClause";
5133     }
5134     if ($this->_permissionWhereClause) {
5135       $where .= " AND " . $this->_permissionWhereClause;
5136     }
5137     if ($context == 'search') {
5138       $where .= " AND contact_a.is_deleted = 0 ";
5139     }
5140
5141     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5142
5143     $summary = ['total' => [], 'soft_credit' => ['count' => 0, 'avg' => 0, 'amount' => 0]];
5144     $this->addBasicStatsToSummary($summary, $where, $from);
5145
5146     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5147       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5148     }
5149
5150     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5151
5152     return $summary;
5153   }
5154
5155   /**
5156    * Append financial ACL limits to the query from & where clauses, if applicable.
5157    *
5158    * @param string $where
5159    * @param string $from
5160    */
5161   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5162     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5163       return;
5164     }
5165     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5166     if (!empty($financialTypes)) {
5167       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5168       $from .= " LEFT JOIN civicrm_line_item li
5169                   ON civicrm_contribution.id = li.contribution_id AND
5170                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5171     }
5172     else {
5173       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5174     }
5175   }
5176
5177   /**
5178    * Getter for the qill object.
5179    *
5180    * @return array
5181    */
5182   public function qill() {
5183     return $this->_qill;
5184   }
5185
5186   /**
5187    * Default set of return default hier return properties.
5188    *
5189    * @return array
5190    */
5191   public static function &defaultHierReturnProperties() {
5192     if (!isset(self::$_defaultHierReturnProperties)) {
5193       self::$_defaultHierReturnProperties = [
5194         'home_URL' => 1,
5195         'image_URL' => 1,
5196         'legal_identifier' => 1,
5197         'external_identifier' => 1,
5198         'contact_type' => 1,
5199         'contact_sub_type' => 1,
5200         'sort_name' => 1,
5201         'display_name' => 1,
5202         'nick_name' => 1,
5203         'first_name' => 1,
5204         'middle_name' => 1,
5205         'last_name' => 1,
5206         'prefix_id' => 1,
5207         'suffix_id' => 1,
5208         'formal_title' => 1,
5209         'communication_style_id' => 1,
5210         'email_greeting' => 1,
5211         'postal_greeting' => 1,
5212         'addressee' => 1,
5213         'birth_date' => 1,
5214         'gender_id' => 1,
5215         'preferred_communication_method' => 1,
5216         'do_not_phone' => 1,
5217         'do_not_email' => 1,
5218         'do_not_mail' => 1,
5219         'do_not_sms' => 1,
5220         'do_not_trade' => 1,
5221         'location' => [
5222           '1' => [
5223             'location_type' => 1,
5224             'street_address' => 1,
5225             'city' => 1,
5226             'state_province' => 1,
5227             'postal_code' => 1,
5228             'postal_code_suffix' => 1,
5229             'country' => 1,
5230             'phone-Phone' => 1,
5231             'phone-Mobile' => 1,
5232             'phone-Fax' => 1,
5233             'phone-1' => 1,
5234             'phone-2' => 1,
5235             'phone-3' => 1,
5236             'im-1' => 1,
5237             'im-2' => 1,
5238             'im-3' => 1,
5239             'email-1' => 1,
5240             'email-2' => 1,
5241             'email-3' => 1,
5242           ],
5243           '2' => [
5244             'location_type' => 1,
5245             'street_address' => 1,
5246             'city' => 1,
5247             'state_province' => 1,
5248             'postal_code' => 1,
5249             'postal_code_suffix' => 1,
5250             'country' => 1,
5251             'phone-Phone' => 1,
5252             'phone-Mobile' => 1,
5253             'phone-1' => 1,
5254             'phone-2' => 1,
5255             'phone-3' => 1,
5256             'im-1' => 1,
5257             'im-2' => 1,
5258             'im-3' => 1,
5259             'email-1' => 1,
5260             'email-2' => 1,
5261             'email-3' => 1,
5262           ],
5263         ],
5264       ];
5265     }
5266     return self::$_defaultHierReturnProperties;
5267   }
5268
5269   /**
5270    * Build query for a date field.
5271    *
5272    * @param array $values
5273    * @param string $tableName
5274    * @param string $fieldName
5275    * @param string $dbFieldName
5276    * @param string $fieldTitle
5277    * @param bool $appendTimeStamp
5278    * @param string $dateFormat
5279    * @param string|null $highDBFieldName
5280    *   Optional field name for when the 'high' part of the calculation uses a different field than the 'low' part.
5281    *   This is an obscure situation & one we don't want to do more of but supporting them here is the only way for now.
5282    *   Examples are event date & relationship active date -in both cases we are looking for things greater than the start
5283    *   date & less than the end date.
5284    *
5285    * @throws \CRM_Core_Exception
5286    */
5287   public function dateQueryBuilder(
5288     $values, $tableName, $fieldName,
5289     $dbFieldName, $fieldTitle,
5290     $appendTimeStamp = TRUE,
5291     $dateFormat = 'YmdHis',
5292     $highDBFieldName = NULL
5293   ) {
5294     // @todo - remove dateFormat - pretty sure it's never passed in...
5295     list($name, $op, $value, $grouping, $wildcard) = $values;
5296     if ($name !== $fieldName && $name !== "{$fieldName}_low" && $name !== "{$fieldName}_high") {
5297       CRM_Core_Error::deprecatedFunctionWarning('Date query builder called unexpectedly');
5298       return;
5299     }
5300     if ($tableName === 'civicrm_contact') {
5301       // Special handling for contact table as it has a known alias in advanced search.
5302       $tableName = 'contact_a';
5303     }
5304     if ($name === "{$fieldName}_low" ||
5305       $name === "{$fieldName}_high"
5306     ) {
5307       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5308         return;
5309       }
5310       $this->_rangeCache[$fieldName] = 1;
5311
5312       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5313
5314       if ($name == $fieldName . '_low') {
5315         $firstOP = '>=';
5316         $firstPhrase = ts('greater than or equal to');
5317         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5318
5319         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5320         if (!empty($secondValues) && $secondValues[2]) {
5321           $secondOP = '<=';
5322           $secondPhrase = ts('less than or equal to');
5323           $secondValue = $secondValues[2];
5324
5325           if ($appendTimeStamp && strlen($secondValue) == 10) {
5326             $secondValue .= ' 23:59:59';
5327           }
5328           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5329         }
5330       }
5331       elseif ($name == $fieldName . '_high') {
5332         $firstOP = '<=';
5333         $firstPhrase = ts('less than or equal to');
5334
5335         if ($appendTimeStamp && strlen($value) == 10) {
5336           $value .= ' 23:59:59';
5337         }
5338         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5339
5340         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5341         if (!empty($secondValues) && $secondValues[2]) {
5342           $secondOP = '>=';
5343           $secondPhrase = ts('greater than or equal to');
5344           $secondValue = $secondValues[2];
5345           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5346         }
5347       }
5348
5349       if (!$appendTimeStamp) {
5350         $firstDate = substr($firstDate, 0, 8);
5351       }
5352       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5353
5354       if ($secondDate) {
5355         if (!$appendTimeStamp) {
5356           $secondDate = substr($secondDate, 0, 8);
5357         }
5358         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5359       }
5360
5361       if ($secondDate) {
5362         $highDBFieldName = $highDBFieldName ?? $dbFieldName;
5363         $this->_where[$grouping][] = "
5364 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5365 ( {$tableName}.{$highDBFieldName} $secondOP '$secondDate' )
5366 ";
5367         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5368       }
5369       else {
5370         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5371         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5372       }
5373     }
5374
5375     if ($name == $fieldName) {
5376       //In Get API, for operators other then '=' the $value is in array(op => value) format
5377       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5378         $op = key($value);
5379         $value = $value[$op];
5380       }
5381
5382       $date = $format = NULL;
5383       if (strstr($op, 'IN')) {
5384         $format = [];
5385         foreach ($value as &$date) {
5386           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5387           if (!$appendTimeStamp) {
5388             $date = substr($date, 0, 8);
5389           }
5390           $format[] = CRM_Utils_Date::customFormat($date);
5391         }
5392         $date = "('" . implode("','", $value) . "')";
5393         $format = implode(', ', $format);
5394       }
5395       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5396         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5397         if (!$appendTimeStamp) {
5398           $date = substr($date, 0, 8);
5399         }
5400         $format = CRM_Utils_Date::customFormat($date);
5401         $date = "'$date'";
5402       }
5403
5404       if ($date) {
5405         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5406       }
5407       else {
5408         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5409       }
5410
5411       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5412       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5413     }
5414
5415     // Ensure the tables are set, but don't whomp anything.
5416     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
5417     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
5418   }
5419
5420   /**
5421    * @param $values
5422    * @param string $tableName
5423    * @param string $fieldName
5424    * @param string $dbFieldName
5425    * @param $fieldTitle
5426    * @param null $options
5427    */
5428   public function numberRangeBuilder(
5429     &$values,
5430     $tableName, $fieldName,
5431     $dbFieldName, $fieldTitle,
5432     $options = NULL
5433   ) {
5434     list($name, $op, $value, $grouping, $wildcard) = $values;
5435
5436     if ($name == "{$fieldName}_low" ||
5437       $name == "{$fieldName}_high"
5438     ) {
5439       if (isset($this->_rangeCache[$fieldName])) {
5440         return;
5441       }
5442       $this->_rangeCache[$fieldName] = 1;
5443
5444       $secondOP = $secondPhrase = $secondValue = NULL;
5445
5446       if ($name == "{$fieldName}_low") {
5447         $firstOP = '>=';
5448         $firstPhrase = ts('greater than');
5449
5450         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5451         if (!empty($secondValues)) {
5452           $secondOP = '<=';
5453           $secondPhrase = ts('less than');
5454           $secondValue = $secondValues[2];
5455         }
5456       }
5457       else {
5458         $firstOP = '<=';
5459         $firstPhrase = ts('less than');
5460
5461         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5462         if (!empty($secondValues)) {
5463           $secondOP = '>=';
5464           $secondPhrase = ts('greater than');
5465           $secondValue = $secondValues[2];
5466         }
5467       }
5468
5469       if ($secondOP) {
5470         $this->_where[$grouping][] = "
5471 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5472 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5473 ";
5474         $displayValue = $options ? $options[$value] : $value;
5475         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5476
5477         $this->_qill[$grouping][]
5478           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5479       }
5480       else {
5481         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5482         $displayValue = $options ? $options[$value] : $value;
5483         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5484       }
5485       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5486
5487       return;
5488     }
5489
5490     if ($name == $fieldName) {
5491       $op = '=';
5492       $phrase = '=';
5493
5494       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5495
5496       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5497       $displayValue = $options ? $options[$value] : $value;
5498       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5499     }
5500   }
5501
5502   /**
5503    * @param $values
5504    * @param string $tableName
5505    * @param string $fieldName
5506    * @param string $dbFieldName
5507    * @param $fieldTitle
5508    * @param null $options
5509    */
5510   public function ageRangeQueryBuilder(
5511     &$values,
5512     $tableName, $fieldName,
5513     $dbFieldName, $fieldTitle,
5514     $options = NULL
5515   ) {
5516     list($name, $op, $value, $grouping, $wildcard) = $values;
5517
5518     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5519     // will be treated as current day
5520     $asofDate = NULL;
5521     if ($asofDateValues) {
5522       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5523       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5524       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5525     }
5526
5527     if ($name == "{$fieldName}_low" ||
5528       $name == "{$fieldName}_high"
5529     ) {
5530       if (isset($this->_rangeCache[$fieldName])) {
5531         return;
5532       }
5533       $this->_rangeCache[$fieldName] = 1;
5534
5535       $secondOP = $secondPhrase = $secondValue = NULL;
5536
5537       if ($name == "{$fieldName}_low") {
5538         $firstPhrase = ts('greater than or equal to');
5539         // NB: age > X means date of birth < Y
5540         $firstOP = '<=';
5541         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5542
5543         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5544         if (!empty($secondValues)) {
5545           $secondOP = '>=';
5546           $secondPhrase = ts('less than or equal to');
5547           $secondValue = $secondValues[2];
5548           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5549         }
5550       }
5551       else {
5552         $firstOP = '>=';
5553         $firstPhrase = ts('less than or equal to');
5554         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5555
5556         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5557         if (!empty($secondValues)) {
5558           $secondOP = '<=';
5559           $secondPhrase = ts('greater than or equal to');
5560           $secondValue = $secondValues[2];
5561           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5562         }
5563       }
5564
5565       if ($secondOP) {
5566         $this->_where[$grouping][] = "
5567 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5568 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5569 ";
5570         $displayValue = $options ? $options[$value] : $value;
5571         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5572
5573         $this->_qill[$grouping][]
5574           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5575       }
5576       else {
5577         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5578         $displayValue = $options ? $options[$value] : $value;
5579         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5580       }
5581       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5582       return;
5583     }
5584   }
5585
5586   /**
5587    * Calculate date from age.
5588    *
5589    * @param string $asofDate
5590    * @param int $age
5591    * @param string $type
5592    *
5593    * @return string
5594    * @throws \Exception
5595    */
5596   public static function calcDateFromAge($asofDate, $age, $type) {
5597     $date = new DateTime($asofDate);
5598     if ($type == "min") {
5599       // minimum age is $age: dob <= date - age "235959"
5600       $date->sub(new DateInterval("P" . $age . "Y"));
5601       return $date->format('Ymd') . "235959";
5602     }
5603     else {
5604       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5605       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5606       return $date->format('Ymd') . "000000";
5607     }
5608   }
5609
5610   /**
5611    * Given the field name, operator, value & its data type
5612    * builds the where Clause for the query
5613    * used for handling 'IS NULL'/'IS NOT NULL' operators
5614    *
5615    * @param string $field
5616    *   Fieldname.
5617    * @param string $op
5618    *   Operator.
5619    * @param string $value
5620    *   Value.
5621    * @param string $dataType
5622    *   Data type of the field.
5623    *
5624    * @return string
5625    *   Where clause for the query.
5626    * @throws \CRM_Core_Exception
5627    */
5628   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5629     $op = trim($op);
5630     $clause = "$field $op";
5631
5632     switch ($op) {
5633       case 'IS NULL':
5634       case 'IS NOT NULL':
5635         return $clause;
5636
5637       case 'IS EMPTY':
5638         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5639         return $clause;
5640
5641       case 'IS NOT EMPTY':
5642         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5643         return $clause;
5644
5645       case 'RLIKE':
5646         return " CAST({$field} AS BINARY) RLIKE BINARY '{$value}' ";
5647
5648       case 'IN':
5649       case 'NOT IN':
5650         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5651         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5652           $value = [$op => (array) $value];
5653         }
5654
5655       default:
5656         if (empty($dataType) || $dataType == 'Date') {
5657           $dataType = 'String';
5658         }
5659         if (is_array($value)) {
5660           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5661           // widely used and consistent across the codebase
5662           // adding this here won't accept the search functions which don't submit an array
5663           if (($queryString = CRM_Core_DAO::createSQLFilter($field, $value, $dataType)) != FALSE) {
5664
5665             return $queryString;
5666           }
5667           if (!empty($value[0]) && $op === 'BETWEEN') {
5668             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5669             if (($queryString = CRM_Core_DAO::createSQLFilter($field, [$op => $value], $dataType)) != FALSE) {
5670               return $queryString;
5671             }
5672           }
5673           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5674         }
5675         $emojiWhere = CRM_Utils_SQL::handleEmojiInQuery($value);
5676         if ($emojiWhere === '0 = 1') {
5677           $value = $emojiWhere;
5678         }
5679         $value = CRM_Utils_Type::escape($value, $dataType);
5680         // if we don't have a dataType we should assume
5681         if ($dataType == 'String' || $dataType == 'Text') {
5682           $value = "'" . $value . "'";
5683         }
5684         return "$clause $value";
5685     }
5686   }
5687
5688   /**
5689    * @param bool $reset
5690    *
5691    * @return array
5692    */
5693   public function openedSearchPanes($reset = FALSE) {
5694     if (!$reset || empty($this->_whereTables)) {
5695       return self::$_openedPanes;
5696     }
5697
5698     // pane name to table mapper
5699     $panesMapper = [
5700       ts('Contributions') => 'civicrm_contribution',
5701       ts('Memberships') => 'civicrm_membership',
5702       ts('Events') => 'civicrm_participant',
5703       ts('Relationships') => 'civicrm_relationship',
5704       ts('Activities') => 'civicrm_activity',
5705       ts('Pledges') => 'civicrm_pledge',
5706       ts('Cases') => 'civicrm_case',
5707       ts('Address Fields') => 'civicrm_address',
5708       ts('Notes') => 'civicrm_note',
5709       ts('Change Log') => 'civicrm_log',
5710       ts('Mailings') => 'civicrm_mailing',
5711     ];
5712     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5713
5714     foreach (array_keys($this->_whereTables) as $table) {
5715       if ($panName = array_search($table, $panesMapper)) {
5716         self::$_openedPanes[$panName] = TRUE;
5717       }
5718     }
5719
5720     return self::$_openedPanes;
5721   }
5722
5723   /**
5724    * @param $operator
5725    */
5726   public function setOperator($operator) {
5727     $validOperators = ['AND', 'OR'];
5728     if (!in_array($operator, $validOperators)) {
5729       $operator = 'AND';
5730     }
5731     $this->_operator = $operator;
5732   }
5733
5734   /**
5735    * @return string
5736    */
5737   public function getOperator() {
5738     return $this->_operator;
5739   }
5740
5741   /**
5742    * @param $from
5743    * @param $where
5744    * @param $having
5745    */
5746   public function filterRelatedContacts(&$from, &$where, &$having) {
5747     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5748       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5749     }
5750     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5751     // since there only can be one instance of this filter in every query
5752     // skip if filter has already applied
5753     foreach ($_rTempCache as $acache) {
5754       foreach ($acache['queries'] as $aqcache) {
5755         if (strpos($from, $aqcache['from']) !== FALSE) {
5756           $having = NULL;
5757           return;
5758         }
5759       }
5760     }
5761     $arg_sig = sha1("$from $where $having");
5762     if (isset($_rTempCache[$arg_sig])) {
5763       $cache = $_rTempCache[$arg_sig];
5764     }
5765     else {
5766       // create temp table with contact ids
5767
5768       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5769
5770       $sql = "
5771 REPLACE INTO $tableName ( contact_id )
5772 SELECT contact_a.id
5773        $from
5774        $where
5775        $having
5776 ";
5777       CRM_Core_DAO::executeQuery($sql);
5778
5779       $cache = ['tableName' => $tableName, 'queries' => []];
5780       $_rTempCache[$arg_sig] = $cache;
5781     }
5782     // upsert the query depending on relationship type
5783     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5784       $qcache = $cache['queries'][$this->_displayRelationshipType];
5785     }
5786     else {
5787       $tableName = $cache['tableName'];
5788       $qcache = [
5789         "from" => "",
5790         "where" => "",
5791       ];
5792       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5793       if (is_numeric($this->_displayRelationshipType)) {
5794         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5795         $qcache['from'] = "
5796 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5797 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5798 ";
5799         $qcache['where'] = "
5800 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5801 AND   displayRelType.is_active = 1
5802 ";
5803       }
5804       else {
5805         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5806         if ($dirOne == 'a') {
5807           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5808           $qcache['from'] .= "
5809 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5810 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5811 ";
5812         }
5813         else {
5814           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5815           $qcache['from'] .= "
5816 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5817 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5818 ";
5819         }
5820         $qcache['where'] = "
5821 WHERE displayRelType.relationship_type_id = $relType
5822 AND   displayRelType.is_active = 1
5823 ";
5824       }
5825       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5826       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5827     }
5828     $qillMessage = ts('Contacts with a Relationship Type of: ');
5829     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5830     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5831       $this->_qill[0][] = $iqill;
5832     }
5833     if (strpos($from, $qcache['from']) === FALSE) {
5834       if (strpos($from, "INNER JOIN") !== FALSE) {
5835         // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5836         // this happens when we have an event_type in the quert (CRM-7969)
5837         $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5838         // Make sure the relationship join right after the FROM and other joins afterwards.
5839         // This gives us the possibility to change the join on civicrm case.
5840         $from = preg_replace("/LEFT JOIN/", $qcache['from'] . " LEFT JOIN", $from, 1);
5841       }
5842       else {
5843         $from .= $qcache['from'];
5844       }
5845       $where = $qcache['where'];
5846       if (!empty($this->_tables['civicrm_case'])) {
5847         // Change the join on CiviCRM case so that it joins on the right contac from the relationship.
5848         $from = str_replace("ON civicrm_case_contact.contact_id = contact_a.id", "ON civicrm_case_contact.contact_id = transform_temp.contact_id", $from);
5849         $where .= " AND displayRelType.case_id = civicrm_case_contact.case_id ";
5850       }
5851       if (!empty($this->_permissionFromClause) && !stripos($from, 'aclContactCache')) {
5852         $from .= " $this->_permissionFromClause";
5853       }
5854       if (!empty($this->_permissionWhereClause)) {
5855         $where .= "AND $this->_permissionWhereClause";
5856       }
5857     }
5858
5859     $having = NULL;
5860   }
5861
5862   /**
5863    * See CRM-19811 for why this is database hurty without apparent benefit.
5864    *
5865    * @param $op
5866    *
5867    * @return bool
5868    */
5869   public static function caseImportant($op) {
5870     return !in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']);
5871   }
5872
5873   /**
5874    * @param $returnProperties
5875    * @param $prefix
5876    *
5877    * @return bool
5878    */
5879   public static function componentPresent(&$returnProperties, $prefix) {
5880     foreach ($returnProperties as $name => $dontCare) {
5881       if (substr($name, 0, strlen($prefix)) == $prefix) {
5882         return TRUE;
5883       }
5884     }
5885     return FALSE;
5886   }
5887
5888   /**
5889    * Builds the necessary structures for all fields that are similar to option value look-ups.
5890    *
5891    * @param string $name
5892    *   the name of the field.
5893    * @param string $op
5894    *   the sql operator, this function should handle ALL SQL operators.
5895    * @param string $value
5896    *   depends on the operator and who's calling the query builder.
5897    * @param int $grouping
5898    *   the index where to place the where clause.
5899    * @param string $daoName
5900    *   DAO Name.
5901    * @param array $field
5902    *   an array that contains various properties of the field identified by $name.
5903    * @param string $label
5904    *   The label for this field element.
5905    * @param string $dataType
5906    *
5907    * @throws \CRM_Core_Exception
5908    */
5909   public function optionValueQuery(
5910     $name,
5911     $op,
5912     $value,
5913     $grouping,
5914     $daoName,
5915     $field,
5916     $label,
5917     $dataType = 'String'
5918   ) {
5919
5920     $pseudoFields = [
5921       'email_greeting',
5922       'postal_greeting',
5923       'addressee',
5924     ];
5925
5926     list($tableName, $fieldName) = explode('.', $field['where'], 2);
5927     if ($tableName == 'civicrm_contact') {
5928       $wc = "contact_a.$fieldName";
5929     }
5930     else {
5931       // Special handling for on_hold, so that we actually use the 'where'
5932       // property in order to limit the query by the on_hold status of the email,
5933       // instead of using email.id which would be nonsensical.
5934       if ($field['name'] === 'on_hold') {
5935         $wc = $field['where'];
5936       }
5937       else {
5938         $wc = "$tableName.id";
5939       }
5940     }
5941
5942     if (in_array($name, $pseudoFields)) {
5943       $wc = "contact_a.{$name}_id";
5944       $dataType = 'Positive';
5945       $value = (!$value) ? 0 : $value;
5946     }
5947     if ($name == "world_region") {
5948       $field['name'] = $name;
5949     }
5950
5951     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5952     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5953     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5954   }
5955
5956   /**
5957    * Check and explode a user defined numeric string into an array
5958    * this was the protocol used by search builder in the old old days before we had
5959    * super nice js widgets to do the hard work
5960    *
5961    * @param string $string
5962    * @param string $dataType
5963    *   The dataType we should check for the values, default integer.
5964    *
5965    * @return bool|array
5966    *   false if string does not match the pattern
5967    *   array of numeric values if string does match the pattern
5968    */
5969   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5970     $string = trim($string);
5971     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5972       return FALSE;
5973     }
5974
5975     $string = substr($string, 1, -1);
5976     $values = explode(',', $string);
5977     if (empty($values)) {
5978       return FALSE;
5979     }
5980
5981     $returnValues = [];
5982     foreach ($values as $v) {
5983       if ($dataType == 'Integer' && !is_numeric($v)) {
5984         return FALSE;
5985       }
5986       elseif ($dataType == 'String' && !is_string($v)) {
5987         return FALSE;
5988       }
5989       $returnValues[] = trim($v);
5990     }
5991
5992     if (empty($returnValues)) {
5993       return FALSE;
5994     }
5995
5996     return $returnValues;
5997   }
5998
5999   /**
6000    * Convert the pseudo constants id's to their names
6001    *
6002    * @param CRM_Core_DAO $dao
6003    * @param bool $return
6004    * @param bool $usedForAPI
6005    *
6006    * @return array|NULL
6007    */
6008   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
6009     if (empty($this->_pseudoConstantsSelect)) {
6010       return NULL;
6011     }
6012     $values = [];
6013     foreach ($this->_pseudoConstantsSelect as $key => $value) {
6014       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
6015         continue;
6016       }
6017
6018       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6019         $val = $dao->{$value['idCol']};
6020         if ($key == 'groups') {
6021           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6022           continue;
6023         }
6024
6025         if (CRM_Utils_System::isNull($val)) {
6026           $dao->$key = NULL;
6027         }
6028         elseif (!empty($value['pseudoconstant'])) {
6029           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6030           // and we can use the metadata to figure it out.
6031           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6032           // more metadata based choices.
6033           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6034             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6035             foreach ($dbValues as $pseudoValue) {
6036               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6037             }
6038
6039             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6040             $realFieldName = $this->_pseudoConstantsSelect[$key]['field_name'] ?? NULL;
6041             if ($usedForAPI && $realFieldName) {
6042               // normally we would see 2 fields returned for pseudoConstants. An exception is
6043               // preferred_communication_method where there is no id-variant.
6044               // For the api we prioritise getting the real data returned.
6045               // over the resolved version
6046               $dao->$realFieldName = $dbValues;
6047             }
6048
6049           }
6050           else {
6051             // This is basically the same as the default but since we have the bao we can use
6052             // a cached function.
6053             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6054           }
6055         }
6056         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6057           //preserve id value
6058           $idColumn = "{$key}_id";
6059           $dao->$idColumn = $val;
6060
6061           if ($key == 'state_province_name') {
6062             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6063           }
6064           else {
6065             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6066           }
6067         }
6068         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6069           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6070         }
6071         // @todo handle this in the section above for pseudoconstants.
6072         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6073           // @todo define bao on this & merge into the above condition.
6074           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6075
6076           if ($value['pseudoField'] == 'participant_role') {
6077             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6078             foreach ($viewValues as $k => $v) {
6079               $viewValues[$k] = $pseudoOptions[$v];
6080             }
6081           }
6082           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6083         }
6084         else {
6085           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6086           $dao->$key = $labels[$val];
6087         }
6088
6089         // return converted values in array format
6090         if ($return) {
6091           if (strpos($key, '-') !== FALSE) {
6092             $keyVal = explode('-', $key);
6093             $current = &$values;
6094             $lastElement = array_pop($keyVal);
6095             foreach ($keyVal as $v) {
6096               if (!array_key_exists($v, $current)) {
6097                 $current[$v] = [];
6098               }
6099               $current = &$current[$v];
6100             }
6101             $current[$lastElement] = $dao->$key;
6102           }
6103           else {
6104             $values[$key] = $dao->$key;
6105           }
6106         }
6107       }
6108     }
6109     if (!$usedForAPI) {
6110       foreach ($this->legacyHackedFields as $realField => $labelField) {
6111         // This is a temporary routine for handling these fields while
6112         // we figure out how to handled them based on metadata in
6113         /// export and search builder. CRM-19815, CRM-19830.
6114         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6115           $dao->$realField = $dao->$labelField;
6116         }
6117       }
6118     }
6119     return $values;
6120   }
6121
6122   /**
6123    * Include pseudo fields LEFT JOIN.
6124    * @param string|array $sort can be a object or string
6125    *
6126    * @return array|NULL
6127    */
6128   public function includePseudoFieldsJoin($sort) {
6129     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6130       return NULL;
6131     }
6132     $sort = is_string($sort) ? $sort : $sort->orderBy();
6133     $present = [];
6134
6135     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6136       if (!empty($value['table'])) {
6137         $regex = "/({$value['table']}\.|{$name})/";
6138         if (preg_match($regex, $sort)) {
6139           $this->_elemnt[$value['element']] = 1;
6140           $this->_select[$value['element']] = $value['select'];
6141           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6142           $present[$value['table']] = $value['join'];
6143         }
6144       }
6145     }
6146     $presentSimpleFrom = $present;
6147
6148     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6149       array_key_exists('civicrm_country', $presentSimpleFrom)
6150     ) {
6151       unset($presentSimpleFrom['civicrm_country']);
6152     }
6153     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6154       array_key_exists('civicrm_country', $present)
6155     ) {
6156       unset($present['civicrm_country']);
6157     }
6158
6159     $presentClause = $presentSimpleFromClause = NULL;
6160     if (!empty($present)) {
6161       $presentClause = implode(' ', $present);
6162     }
6163     if (!empty($presentSimpleFrom)) {
6164       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6165     }
6166
6167     $this->_fromClause = $this->_fromClause . $presentClause;
6168     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6169
6170     return [$presentClause, $presentSimpleFromClause];
6171   }
6172
6173   /**
6174    * Build qill for field.
6175    *
6176    * Qill refers to the query detail visible on the UI.
6177    *
6178    * @param string $daoName
6179    * @param string $fieldName
6180    * @param mixed $fieldValue
6181    * @param string $op
6182    * @param array $pseudoExtraParam
6183    * @param int $type
6184    *   Type of the field per CRM_Utils_Type
6185    *
6186    * @return array
6187    */
6188   public static function buildQillForFieldValue(
6189     $daoName,
6190     $fieldName,
6191     $fieldValue,
6192     $op,
6193     $pseudoExtraParam = [],
6194     $type = CRM_Utils_Type::T_STRING
6195   ) {
6196     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6197
6198     //API usually have fieldValue format as array(operator => array(values)),
6199     //so we need to separate operator out of fieldValue param
6200     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6201       $op = key($fieldValue);
6202       $fieldValue = $fieldValue[$op];
6203     }
6204
6205     // if Operator chosen is NULL/EMPTY then
6206     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6207       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6208     }
6209
6210     // @todo - if the right BAO is passed in special handling for the below
6211     // fields should not be required. testQillOptions.
6212     if ($fieldName == 'country_id') {
6213       $pseudoOptions = CRM_Core_PseudoConstant::country();
6214     }
6215     elseif ($fieldName == 'county_id') {
6216       $pseudoOptions = CRM_Core_PseudoConstant::county();
6217     }
6218     elseif ($fieldName == 'world_region') {
6219       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6220     }
6221     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6222       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6223       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6224     }
6225     elseif ($fieldName == 'contribution_product_id') {
6226       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6227     }
6228     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6229       $pseudoOptions = CRM_Core_PseudoConstant::group();
6230     }
6231     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6232       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6233     }
6234     elseif ($daoName) {
6235       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6236     }
6237
6238     if (is_array($fieldValue)) {
6239       $qillString = [];
6240       if (!empty($pseudoOptions)) {
6241         foreach ((array) $fieldValue as $val) {
6242           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6243         }
6244         $fieldValue = implode(', ', $qillString);
6245       }
6246       else {
6247         if ($type == CRM_Utils_Type::T_DATE) {
6248           foreach ($fieldValue as $index => $value) {
6249             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6250           }
6251         }
6252         $separator = ', ';
6253         // @todo - this is a bit specific (one operator).
6254         // However it is covered by a unit test so can be altered later with
6255         // some confidence.
6256         if ($op === 'BETWEEN') {
6257           $separator = ' AND ';
6258         }
6259         $fieldValue = implode($separator, $fieldValue);
6260       }
6261     }
6262     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6263       $fieldValue = $pseudoOptions[$fieldValue];
6264     }
6265     elseif ($type === CRM_Utils_Type::T_DATE) {
6266       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6267     }
6268
6269     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6270   }
6271
6272   /**
6273    * Get the qill (search description for field) for the specified field.
6274    *
6275    * @param string $daoName
6276    * @param string $name
6277    * @param string $value
6278    * @param string|array $op
6279    * @param string $label
6280    *
6281    * @return string
6282    */
6283   public static function getQillValue($daoName, string $name, $value, $op, string $label) {
6284     list($op, $value) = self::buildQillForFieldValue($daoName, $name, $value, $op);
6285     return ts('%1 %2 %3', [1 => $label, 2 => $op, 3 => $value]);
6286   }
6287
6288   /**
6289    * Alter value to reflect wildcard settings.
6290    *
6291    * The form will have tried to guess whether this is a good field to wildcard but there is
6292    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6293    * or only the end of the string
6294    *
6295    * @param bool $wildcard
6296    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6297    * @param string $op
6298    *   Generally '=' or 'LIKE'.
6299    * @param string $value
6300    *   The search string.
6301    *
6302    * @return string
6303    */
6304   public static function getWildCardedValue($wildcard, $op, $value) {
6305     if ($wildcard && $op === 'LIKE') {
6306       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6307         return "%$value%";
6308       }
6309       else {
6310         return "$value%";
6311       }
6312     }
6313     else {
6314       return "$value";
6315     }
6316   }
6317
6318   /**
6319    * Process special fields of Search Form in OK (Operator in Key) format
6320    *
6321    * @param array $formValues
6322    * @param array $specialFields
6323    *    Special params to be processed
6324    * @param array $changeNames
6325    *   Array of fields whose name should be changed
6326    */
6327   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6328     // Array of special fields whose value are considered only for NULL or EMPTY operators
6329     $nullableFields = ['contribution_batch_id'];
6330
6331     foreach ($specialFields as $element) {
6332       $value = $formValues[$element] ?? NULL;
6333       if ($value) {
6334         if (is_array($value)) {
6335           if (array_key_exists($element, $changeNames)) {
6336             unset($formValues[$element]);
6337             $element = $changeNames[$element];
6338           }
6339           $formValues[$element] = ['IN' => $value];
6340         }
6341         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6342           $formValues[$element] = [$value => 1];
6343         }
6344         elseif (!in_array($element, $nullableFields)) {
6345           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6346           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6347           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6348         }
6349       }
6350     }
6351   }
6352
6353   /**
6354    * Parse and assimilate the various sort options.
6355    *
6356    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6357    * `email`), the related table may be joined automatically.
6358    *
6359    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6360    * it's been extracted to a standalone function.
6361    *
6362    * @param string|CRM_Utils_Sort $sort
6363    *   The order by string.
6364    * @param null $sortOrder
6365    *   Who knows? Hu knows. He who knows Hu knows who.
6366    *
6367    * @return string
6368    *   list(string $orderByClause, string $additionalFromClause).
6369    *
6370    * @throws \CRM_Core_Exception
6371    */
6372   protected function prepareOrderBy($sort, $sortOrder) {
6373     $orderByArray = [];
6374     $orderBy = '';
6375
6376     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6377       isset($this->_distinctComponentClause)
6378     ) {
6379       if ($sort) {
6380         if (is_string($sort)) {
6381           $orderBy = $sort;
6382         }
6383         else {
6384           $orderBy = trim($sort->orderBy());
6385         }
6386         // Deliberately remove the backticks again, as they mess up the evil
6387         // string munging below. This balanced by re-escaping before use.
6388         $orderBy = str_replace('`', '', $orderBy);
6389
6390         if (!empty($orderBy)) {
6391           // this is special case while searching for
6392           // change log CRM-1718
6393           if (preg_match('/sort_name/i', $orderBy)) {
6394             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6395           }
6396
6397           if ($sortOrder) {
6398             $orderBy .= " $sortOrder";
6399           }
6400
6401           // always add contact_a.id to the ORDER clause
6402           // so the order is deterministic
6403           if (strpos('contact_a.id', $orderBy) === FALSE) {
6404             $orderBy .= ", contact_a.id";
6405           }
6406         }
6407       }
6408       else {
6409         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6410       }
6411     }
6412     if (!$orderBy) {
6413       return NULL;
6414     }
6415     // Remove this here & add it at the end for simplicity.
6416     $order = trim($orderBy);
6417     $orderByArray = explode(',', $order);
6418
6419     foreach ($orderByArray as $orderByClause) {
6420       $orderByClauseParts = explode(' ', trim($orderByClause));
6421       $field = $orderByClauseParts[0];
6422       $direction = $orderByClauseParts[1] ?? 'asc';
6423       $fieldSpec = $this->getMetadataForRealField($field);
6424
6425       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6426       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6427       // add keys to $this->fields to represent the actual keys for locations.
6428       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6429         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6430         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6431       }
6432
6433       if ($this->_returnProperties === []) {
6434         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6435           $this->_tables[$fieldSpec['table_name']] = 1;
6436           $order = $fieldSpec['where'] . ' ' . $direction;
6437         }
6438
6439       }
6440       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6441       // add to cfIDs array if not present
6442       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6443         $this->_cfIDs[$cfID] = [];
6444         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6445         $this->_customQuery->query();
6446         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6447         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6448       }
6449
6450       // By replacing the join to the option value table with the mysql construct
6451       // ORDER BY field('contribution_status_id', 2,1,4)
6452       // we can remove a join. In the case of the option value join it is
6453       /// a join known to cause slow queries.
6454       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6455       // matching an id+name parrern in the
6456       // first instance for scope reasons. They require slightly different handling as the column (label)
6457       // is not declared for them.
6458       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6459       // escaping.
6460       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6461       if (!empty($pseudoConstantMetadata)
6462       ) {
6463         if (!empty($pseudoConstantMetadata['optionGroupName'])
6464           || $this->isPseudoFieldAnFK($fieldSpec)
6465         ) {
6466           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6467           if (empty($fieldSpec['bao'])) {
6468             continue;
6469           }
6470           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name']);
6471           natcasesort($sortedOptions);
6472           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6473           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6474           // this might give some early warning.
6475           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6476           // use where if it's set to fully qualify ambiguous column names
6477           // i.e. civicrm_contribution.contribution_status_id instead of contribution_status_id
6478           $pseudoColumnName = $fieldSpec['where'] ?? $fieldSpec['name'];
6479           $order = str_replace("$field", "field($pseudoColumnName,$fieldIDsInOrder)", $order);
6480         }
6481         //CRM-12565 add "`" around $field if it is a pseudo constant
6482         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6483         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6484           $order = str_replace($field, "`{$field}`", $order);
6485         }
6486       }
6487     }
6488
6489     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6490     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6491
6492     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6493     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6494     if ($order) {
6495       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6496       return ' ORDER BY ' . $order;
6497     }
6498   }
6499
6500   /**
6501    * Convert a string of group IDs to a string of group labels.
6502    *
6503    * The original string may include duplicates and groups the user does not have
6504    * permission to see.
6505    *
6506    * @param CRM_Core_DAO $dao
6507    * @param string $val
6508    *
6509    * @return string
6510    */
6511   public function convertGroupIDStringToLabelString(&$dao, $val) {
6512     $groupIDs = explode(',', $val);
6513     // Note that groups that the user does not have permission to will be excluded (good).
6514     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6515     return implode(', ', $groups);
6516
6517   }
6518
6519   /**
6520    * Set the qill and where properties for a field.
6521    *
6522    * This function is intended as a short-term function to encourage refactoring
6523    * & re-use - but really we should just have less special-casing.
6524    *
6525    * @param string $name
6526    * @param string $op
6527    * @param string|array $value
6528    * @param string $grouping
6529    * @param array $field
6530    *
6531    * @throws \CRM_Core_Exception
6532    */
6533   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6534     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6535     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6536     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6537       1 => $field['title'],
6538       2 => $qillop,
6539       3 => $qillVal,
6540     ]);
6541   }
6542
6543   /**
6544    * Has the pseudoconstant of the field been requested.
6545    *
6546    * For example if the field is payment_instrument_id then it
6547    * has been requested if either payment_instrument_id or payment_instrument
6548    * have been requested. Payment_instrument is the option groun name field value.
6549    *
6550    * @param array $field
6551    * @param string $fieldName
6552    *   The unique name of the field - ie. the one it will be aliased to in the query.
6553    *
6554    * @return bool
6555    */
6556   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6557     $realField = $this->getMetadataForRealField($fieldName);
6558     if (!isset($realField['pseudoconstant'])) {
6559       return FALSE;
6560     }
6561     $pseudoConstant = $realField['pseudoconstant'];
6562     if (empty($pseudoConstant['optionGroupName']) &&
6563       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6564       // We are increasing our pseudoconstant handling - but still very cautiously,
6565       // hence the check for labelColumn === name
6566       return FALSE;
6567     }
6568
6569     if (!empty($pseudoConstant['optionGroupName']) && !empty($this->_returnProperties[$pseudoConstant['optionGroupName']])) {
6570       return TRUE;
6571     }
6572     if (!empty($this->_returnProperties[$fieldName])) {
6573       return TRUE;
6574     }
6575     // Is this still required - the above goes off the unique name. Test with things like
6576     // communication_preferences & prefix_id.
6577     if (!empty($this->_returnProperties[$field['name']])) {
6578       return TRUE;
6579     }
6580     return FALSE;
6581   }
6582
6583   /**
6584    * Get Select Clause.
6585    *
6586    * @return string
6587    */
6588   public function getSelect() {
6589     $select = 'SELECT ';
6590     if (isset($this->_distinctComponentClause)) {
6591       $select .= "{$this->_distinctComponentClause}, ";
6592     }
6593     $select .= implode(', ', $this->_select);
6594     return $select;
6595   }
6596
6597   /**
6598    * Add basic statistics to the summary.
6599    *
6600    * @param array $summary
6601    * @param string $where
6602    * @param string $from
6603    *
6604    * @return array
6605    * @throws \CRM_Core_Exception
6606    */
6607   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6608     $summary['total']['count'] = 0;
6609     $summary['total']['amount'] = $summary['total']['avg'] = [];
6610
6611     $query = "
6612       SELECT COUNT( conts.total_amount ) as total_count,
6613         SUM(   conts.total_amount ) as total_amount,
6614         AVG(   conts.total_amount ) as total_avg,
6615         conts.currency              as currency
6616       FROM (
6617         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6618         civicrm_contribution.currency
6619         $from
6620         $where AND civicrm_contribution.contribution_status_id = 1
6621       GROUP BY civicrm_contribution.id
6622     ) as conts
6623     GROUP BY currency";
6624
6625     $dao = CRM_Core_DAO::executeQuery($query);
6626
6627     while ($dao->fetch()) {
6628       $summary['total']['count'] += $dao->total_count;
6629       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6630       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6631     }
6632
6633     if (!empty($summary['total']['amount'])) {
6634       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6635       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6636     }
6637     else {
6638       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6639     }
6640     return $summary;
6641   }
6642
6643   /**
6644    * Add basic soft credit statistics to summary array.
6645    *
6646    * @param array $summary
6647    * @param string $where
6648    * @param string $from
6649    *
6650    * @throws \CRM_Core_Exception
6651    */
6652   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6653     $query = "
6654       SELECT COUNT( conts.total_amount ) as total_count,
6655       SUM(   conts.total_amount ) as total_amount,
6656       AVG(   conts.total_amount ) as total_avg,
6657       conts.currency as currency
6658       FROM (
6659         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6660         $from
6661         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6662         GROUP BY civicrm_contribution_soft.id
6663       ) as conts
6664       GROUP BY currency";
6665
6666     $dao = CRM_Core_DAO::executeQuery($query);
6667     $summary['soft_credit']['count'] = 0;
6668     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6669     while ($dao->fetch()) {
6670       $summary['soft_credit']['count'] += $dao->total_count;
6671       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6672       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6673     }
6674     if (!empty($summary['soft_credit']['amount'])) {
6675       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6676       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6677     }
6678     else {
6679       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6680     }
6681   }
6682
6683   /**
6684    * Add basic stats about cancelled contributions to the summary.
6685    *
6686    * @param array $summary
6687    * @param string $where
6688    * @param string $from
6689    *
6690    * @throws \CRM_Core_Exception
6691    */
6692   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6693     $query = "
6694       SELECT COUNT( conts.total_amount ) as cancel_count,
6695        SUM(   conts.total_amount ) as cancel_amount,
6696        AVG(   conts.total_amount ) as cancel_avg,
6697        conts.currency              as currency
6698         FROM (
6699       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6700       $from
6701       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6702       GROUP BY civicrm_contribution.id
6703     ) as conts
6704     GROUP BY currency";
6705
6706     $dao = CRM_Core_DAO::executeQuery($query);
6707     $summary['cancel'] = ['count' => 0, 'amount' => 0, 'avg' => 0];
6708     if ($dao->N <= 1) {
6709       if ($dao->fetch()) {
6710         $summary['cancel']['count'] = $dao->cancel_count;
6711         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6712         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6713       }
6714     }
6715     else {
6716       $summary['cancel']['count'] = 0;
6717       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6718       while ($dao->fetch()) {
6719         $summary['cancel']['count'] += $dao->cancel_count;
6720         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6721         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6722       }
6723       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6724       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6725     }
6726   }
6727
6728   /**
6729    * Create the sql query for an contact search.
6730    *
6731    * @param int $offset
6732    *   The offset for the query.
6733    * @param int $rowCount
6734    *   The number of rows to return.
6735    * @param string|CRM_Utils_Sort $sort
6736    *   The order by string.
6737    * @param bool $count
6738    *   Is this a count only query ?.
6739    * @param bool $includeContactIds
6740    *   Should we include contact ids?.
6741    * @param bool $sortByChar
6742    *   If true returns the distinct array of first characters for search results.
6743    * @param bool $groupContacts
6744    *   If true, return only the contact ids.
6745    * @param string $additionalWhereClause
6746    *   If the caller wants to further restrict the search (used for components).
6747    * @param null $sortOrder
6748    * @param string $additionalFromClause
6749    *   Should be clause with proper joins, effective to reduce where clause load.
6750    *
6751    * @param bool $skipOrderAndLimit
6752    *
6753    * @return string
6754    *
6755    * @throws \CRM_Core_Exception
6756    */
6757   public function getSearchSQL(
6758     $offset = 0, $rowCount = 0, $sort = NULL,
6759     $count = FALSE, $includeContactIds = FALSE,
6760     $sortByChar = FALSE, $groupContacts = FALSE,
6761     $additionalWhereClause = NULL, $sortOrder = NULL,
6762     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6763
6764     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6765
6766     if ($sortByChar) {
6767       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6768       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6769     }
6770
6771     if ($skipOrderAndLimit) {
6772       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6773       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6774     }
6775     else {
6776       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6777     }
6778     return $query;
6779   }
6780
6781   /**
6782    * Get the component parts of the search query as an array.
6783    *
6784    * @param int $offset
6785    *   The offset for the query.
6786    * @param int $rowCount
6787    *   The number of rows to return.
6788    * @param string|CRM_Utils_Sort $sort
6789    *   The order by string.
6790    * @param bool $count
6791    *   Is this a count only query ?.
6792    * @param bool $includeContactIds
6793    *   Should we include contact ids?.
6794    * @param bool $sortByChar
6795    *   If true returns the distinct array of first characters for search results.
6796    * @param bool $groupContacts
6797    *   If true, return only the contact ids.
6798    * @param string $additionalWhereClause
6799    *   If the caller wants to further restrict the search (used for components).
6800    * @param null $sortOrder
6801    * @param string $additionalFromClause
6802    *   Should be clause with proper joins, effective to reduce where clause load.
6803    *
6804    * @return array
6805    * @throws \CRM_Core_Exception
6806    */
6807   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6808     $count = FALSE, $includeContactIds = FALSE,
6809     $sortByChar = FALSE, $groupContacts = FALSE,
6810     $additionalWhereClause = NULL, $sortOrder = NULL,
6811     $additionalFromClause = NULL) {
6812     if ($includeContactIds) {
6813       $this->_includeContactIds = TRUE;
6814       $this->_whereClause = $this->whereClause();
6815     }
6816     $onlyDeleted = in_array([
6817       'deleted_contacts',
6818       '=',
6819       '1',
6820       '0',
6821       '0',
6822     ], $this->_params);
6823
6824     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6825     // and that contact happens to be deleted, set $onlyDeleted to true
6826     foreach ($this->_params as $values) {
6827       $name = $values[0] ?? NULL;
6828       $op = $values[1] ?? NULL;
6829       $value = $values[2] ?? NULL;
6830       if ($name === 'contact_id' and $op === '=') {
6831         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6832           $onlyDeleted = TRUE;
6833         }
6834         break;
6835       }
6836     }
6837
6838     // building the query string
6839     $groupBy = $groupByCols = NULL;
6840     if (!$count) {
6841       if (isset($this->_groupByComponentClause)) {
6842         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6843         $groupByCols = explode(', ', $groupByCols);
6844       }
6845       elseif ($this->_useGroupBy) {
6846         $groupByCols = ['contact_a.id'];
6847       }
6848     }
6849     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6850       $groupByCols = ['civicrm_activity.id'];
6851     }
6852     if (!empty($groupByCols)) {
6853       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6854     }
6855
6856     $order = $orderBy = '';
6857     if (!$count) {
6858       if (!$sortByChar) {
6859         $order = $this->prepareOrderBy($sort, $sortOrder);
6860       }
6861     }
6862     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6863     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6864     //     clause. This will impact the search query output.
6865     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6866
6867     if ($disableFullGroupByMode) {
6868       CRM_Core_DAO::disableFullGroupByMode();
6869     }
6870
6871     // CRM-15231
6872     $this->_sort = $sort;
6873
6874     //CRM-15967
6875     $this->includePseudoFieldsJoin($sort);
6876
6877     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6878
6879     if ($additionalWhereClause) {
6880       $where = $where . ' AND ' . $additionalWhereClause;
6881     }
6882
6883     //additional from clause should be w/ proper joins.
6884     if ($additionalFromClause) {
6885       $from .= "\n" . $additionalFromClause;
6886     }
6887
6888     // if we are doing a transform, do it here
6889     // use the $from, $where and $having to get the contact ID
6890     if ($this->_displayRelationshipType) {
6891       $this->filterRelatedContacts($from, $where, $having);
6892     }
6893     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6894
6895     return [
6896       'select' => $select,
6897       'from' => $from,
6898       'where' => $where,
6899       'order_by' => $order,
6900       'group_by' => $groupBy,
6901       'having' => $having,
6902       'limit' => $limit,
6903     ];
6904   }
6905
6906   /**
6907    * Get the metadata for a given field.
6908    *
6909    * @param string $fieldName
6910    *
6911    * @return array
6912    */
6913   protected function getMetadataForField($fieldName) {
6914     if ($fieldName === 'contact_a.id') {
6915       // This seems to be the only anomaly.
6916       $fieldName = 'id';
6917     }
6918     $pseudoField = $this->_pseudoConstantsSelect[$fieldName] ?? [];
6919     $field = $this->_fields[$fieldName] ?? $pseudoField;
6920     $field = array_merge($field, $pseudoField);
6921     if (!empty($field) && empty($field['name'])) {
6922       // standardising field formatting here - over time we can phase out variants.
6923       // all paths using this currently unit tested
6924       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6925     }
6926     return $field;
6927   }
6928
6929   /**
6930    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6931    *
6932    * @param string $fieldName
6933    *
6934    * @return array
6935    */
6936   public function getMetadataForRealField($fieldName) {
6937     $field = $this->getMetadataForField($fieldName);
6938     if (!empty($field['is_pseudofield_for'])) {
6939       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6940       $field['pseudofield_name'] = $fieldName;
6941     }
6942     elseif (!empty($field['pseudoconstant'])) {
6943       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6944         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6945         if (empty($field['table_name'])) {
6946           if (!empty($field['where'])) {
6947             $field['table_name'] = explode('.', $field['where'])[0];
6948           }
6949           else {
6950             $field['table_name'] = 'civicrm_contact';
6951           }
6952         }
6953       }
6954     }
6955     return $field;
6956   }
6957
6958   /**
6959    * Get the field datatype, using the type in the database rather than the pseudofield, if a pseudofield.
6960    *
6961    * @param string $fieldName
6962    *
6963    * @return string
6964    */
6965   public function getDataTypeForRealField($fieldName) {
6966     return CRM_Utils_Type::typeToString($this->getMetadataForRealField($fieldName)['type']);
6967   }
6968
6969   /**
6970    * If we have a field that is better rendered via the pseudoconstant handled them here.
6971    *
6972    * Rather than joining in the additional table we render the option value on output.
6973    *
6974    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6975    * carefully with test coverage but aim to extend.
6976    *
6977    * @param string $name
6978    */
6979   protected function addPseudoconstantFieldToSelect($name) {
6980     $field = $this->getMetadataForRealField($name);
6981     $realFieldName = $field['name'];
6982     $pseudoFieldName = $field['pseudofield_name'] ?? NULL;
6983     if ($pseudoFieldName) {
6984       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6985       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6986         'pseudoField' => $pseudoFieldName,
6987         'idCol' => $realFieldName,
6988         'field_name' => $field['name'],
6989         'bao' => $field['bao'],
6990         'pseudoconstant' => $field['pseudoconstant'],
6991       ];
6992     }
6993
6994     $this->_tables[$field['table_name']] = 1;
6995     $this->_element[$realFieldName] = 1;
6996     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6997   }
6998
6999   /**
7000    * Is this pseudofield a foreign key constraint.
7001    *
7002    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
7003    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
7004    * array which is done for contributions which are mostly handled as pseudoconstants.
7005    *
7006    * @param $fieldSpec
7007    *
7008    * @return bool
7009    */
7010   protected function isPseudoFieldAnFK($fieldSpec) {
7011     if (empty($fieldSpec['FKClassName'])
7012       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
7013       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
7014       return FALSE;
7015     }
7016     return TRUE;
7017   }
7018
7019   /**
7020    * Is the field a relative date field.
7021    *
7022    * @param string $fieldName
7023    *
7024    * @return bool
7025    */
7026   protected function isARelativeDateField($fieldName) {
7027     if (substr($fieldName, -9, 9) !== '_relative') {
7028       return FALSE;
7029     }
7030     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
7031     return isset($this->_fields[$realField]);
7032   }
7033
7034   /**
7035    * Get the specifications for the field, if available.
7036    *
7037    * @param string $fieldName
7038    *   Fieldname as displayed on the form.
7039    *
7040    * @return array
7041    */
7042   public function getFieldSpec($fieldName) {
7043     if (isset($this->_fields[$fieldName])) {
7044       $fieldSpec = $this->_fields[$fieldName];
7045       if (!empty($fieldSpec['is_pseudofield_for'])) {
7046         $fieldSpec = array_merge($this->_fields[$fieldSpec['is_pseudofield_for']], $this->_fields[$fieldName]);
7047       }
7048       return $fieldSpec;
7049     }
7050     $lowFieldName = str_replace('_low', '', $fieldName);
7051     if (isset($this->_fields[$lowFieldName])) {
7052       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
7053     }
7054     $highFieldName = str_replace('_high', '', $fieldName);
7055     if (isset($this->_fields[$highFieldName])) {
7056       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
7057     }
7058     return [];
7059   }
7060
7061   public function buildWhereForDate() {
7062
7063   }
7064
7065   /**
7066    * Is the field a relative date field.
7067    *
7068    * @param string $fieldName
7069    *
7070    * @return bool
7071    */
7072   protected function isADateRangeField($fieldName) {
7073     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7074       return FALSE;
7075     }
7076     return !empty($this->getFieldSpec($fieldName));
7077   }
7078
7079   /**
7080    * @param $values
7081    */
7082   protected function buildRelativeDateQuery(&$values) {
7083     $value = $values[2] ?? NULL;
7084     if (empty($value)) {
7085       return;
7086     }
7087     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7088     $fieldSpec = $this->_fields[$fieldName];
7089     $tableName = $fieldSpec['table_name'];
7090     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7091     $grouping = $values[3] ?? NULL;
7092     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7093     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7094     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7095
7096     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7097     // Where end would be populated only if we are handling one of the weird ones with different from & to fields.
7098     $secondWhere = $fieldSpec['where_end'] ?? $fieldSpec['where'];
7099
7100     $where = $fieldSpec['where'];
7101     if ($fieldSpec['table_name'] === 'civicrm_contact') {
7102       // Special handling for contact table as it has a known alias in advanced search.
7103       $where = str_replace('civicrm_contact.', 'contact_a.', $where);
7104       $secondWhere = str_replace('civicrm_contact.', 'contact_a.', $secondWhere);
7105     }
7106
7107     $this->_qill[$grouping][] = $this->getQillForRelativeDateRange($dates[0], $dates[1], $fieldSpec['title'], $filters[$value]);
7108     if ($fieldName === 'relation_active_period_date') {
7109       // Hack this to fix regression https://lab.civicrm.org/dev/core/issues/1592
7110       // Not sure the  'right' fix.
7111       $this->_where[$grouping] = [self::getRelationshipActivePeriodClauses($dates[0], $dates[1], TRUE)];
7112       return;
7113     }
7114
7115     if (empty($dates[0])) {
7116       // ie. no start date we only have end date
7117       $this->_where[$grouping][] = $secondWhere . " <= '{$dates[1]}'";
7118     }
7119     elseif (empty($dates[1])) {
7120
7121       // ie. no end date we only have start date
7122       $this->_where[$grouping][] = $where . " >= '{$dates[0]}'";
7123     }
7124     else {
7125       // we have start and end dates.
7126       if ($secondWhere !== $where) {
7127         $this->_where[$grouping][] = $where . ">=  '{$dates[0]}' AND $secondWhere <='{$dates[1]}'";
7128       }
7129       else {
7130         $this->_where[$grouping][] = $where . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7131       }
7132     }
7133   }
7134
7135   /**
7136    * Build the query for a date field if it is a _high or _low field.
7137    *
7138    * @param $values
7139    *
7140    * @return bool
7141    * @throws \CRM_Core_Exception
7142    */
7143   public function buildDateRangeQuery($values) {
7144     if ($this->isADateRangeField($values[0])) {
7145       $fieldSpec = $this->getFieldSpec($values[0]);
7146       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7147       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7148       return TRUE;
7149     }
7150     return FALSE;
7151   }
7152
7153   /**
7154    * Add the address table into the query.
7155    *
7156    * @param string $tableKey
7157    * @param string $joinCondition
7158    *
7159    * @return array
7160    *   - alias name
7161    *   - address join.
7162    */
7163   protected function addAddressTable($tableKey, $joinCondition) {
7164     $tName = "$tableKey-address";
7165     $aName = "`$tableKey-address`";
7166     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7167     $this->_element["{$tName}_id"] = 1;
7168     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7169     $this->_tables[$tName] = $addressJoin;
7170
7171     return [
7172       $aName,
7173       $addressJoin,
7174     ];
7175   }
7176
7177   /**
7178    * Get the clause for group status.
7179    *
7180    * @param int $grouping
7181    *
7182    * @return string
7183    */
7184   protected function getGroupStatusClause($grouping) {
7185     $statuses = $this->getSelectedGroupStatuses($grouping);
7186     return "status IN (" . implode(', ', $statuses) . ")";
7187   }
7188
7189   /**
7190    * Get an array of the statuses that have been selected.
7191    *
7192    * @param string $grouping
7193    *
7194    * @return array
7195    *
7196    * @throws \CRM_Core_Exception
7197    */
7198   protected function getSelectedGroupStatuses($grouping) {
7199     $statuses = [];
7200     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7201     if ($gcsValues &&
7202       is_array($gcsValues[2])
7203     ) {
7204       foreach ($gcsValues[2] as $k => $v) {
7205         if ($v) {
7206           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7207         }
7208       }
7209     }
7210     else {
7211       $statuses[] = "'Added'";
7212     }
7213     return $statuses;
7214   }
7215
7216   /**
7217    * Get the qill value for the field.
7218    *
7219    * @param string $name
7220    * @param array|int|string $value
7221    * @param string $op
7222    * @param array $fieldSpec
7223    * @param string $labelOverride
7224    *   Label override, if required.
7225    *
7226    * @return string
7227    */
7228   public function getQillForField($name, $value, $op, $fieldSpec = [], $labelOverride = NULL): string {
7229     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($fieldSpec['bao'] ?? NULL, $name, $value, $op);
7230     return (string) ts("%1 %2 %3", [
7231       1 => $labelOverride ?? $fieldSpec['title'],
7232       2 => $qillop,
7233       3 => $qillVal,
7234     ]);
7235   }
7236
7237   /**
7238    * Where handling for any field with adequately defined metadata.
7239    *
7240    * @param array $fieldSpec
7241    * @param string $name
7242    * @param string|array|int $value
7243    * @param string $op
7244    * @param string|int $grouping
7245    *
7246    * @throws \CRM_Core_Exception
7247    */
7248   public function handleWhereFromMetadata($fieldSpec, $name, $value, $op, $grouping = 0) {
7249     $this->_where[$grouping][] = CRM_Contact_BAO_Query::buildClause($fieldSpec['where'], $op, $value, CRM_Utils_Type::typeToString($fieldSpec['type']));
7250     $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op, $fieldSpec);
7251     if (!isset($this->_tables[$fieldSpec['table_name']])) {
7252       $this->_tables[$fieldSpec['table_name']] = 1;
7253     }
7254     if (!isset($this->_whereTables[$fieldSpec['table_name']])) {
7255       $this->_whereTables[$fieldSpec['table_name']] = 1;
7256     }
7257   }
7258
7259   /**
7260    * Get the qill for the relative date range.
7261    *
7262    * @param string|null $from
7263    * @param string|null $to
7264    * @param string $fieldTitle
7265    * @param string $relativeRange
7266    *
7267    * @return string
7268    */
7269   protected function getQillForRelativeDateRange($from, $to, string $fieldTitle, string $relativeRange): string {
7270     if (!$from) {
7271       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('to %1', [CRM_Utils_Date::customFormat($to)]) . ')';
7272     }
7273     if (!$to) {
7274       return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('from %1', [CRM_Utils_Date::customFormat($from)]) . ')';
7275     }
7276     return ts('%1 is ', [$fieldTitle]) . $relativeRange . ' (' . ts('between %1 and %2', [
7277       CRM_Utils_Date::customFormat($from),
7278       CRM_Utils_Date::customFormat($to),
7279     ]) . ')';
7280   }
7281
7282 }