CRM/Contact/BAO/Query.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33
  34 /**
  35  * This is the heart of the search query building mechanism.
  36  */
  37 class CRM_Contact_BAO_Query {
  38
  39   /**
  40    * The various search modes.
  41    *
  42    * As of February 2017, entries not present for 4, 32, 64, 1024.
  43    *
  44    * MODE_ALL seems to be out of sync with the available constants;
  45    * if this is intentionally excluding MODE_MAILING then that may
  46    * bear documenting?
  47    *
  48    * Likewise if there's reason for the missing modes (4, 32, 64 etc).
  49    *
  50    * @var int
  51    */
  52   const
  53     NO_RETURN_PROPERTIES = 'CRM_Contact_BAO_Query::NO_RETURN_PROPERTIES',
  54     MODE_CONTACTS = 1,
  55     MODE_CONTRIBUTE = 2,
  56     // There is no 4,
  57     MODE_MEMBER = 8,
  58     MODE_EVENT = 16,
  59     MODE_CONTACTSRELATED = 32,
  60     // no 64.
  61     MODE_GRANT = 128,
  62     MODE_PLEDGEBANK = 256,
  63     MODE_PLEDGE = 512,
  64     // There is no 1024,
  65     MODE_CASE = 2048,
  66     MODE_ACTIVITY = 4096,
  67     MODE_CAMPAIGN = 8192,
  68     MODE_MAILING = 16384,
  69     MODE_ALL = 17407;
  70
  71   /**
  72    * Constants for search operators
  73    */
  74   const
  75     SEARCH_OPERATOR_AND = 'AND',
  76     SEARCH_OPERATOR_OR = 'OR';
  77
  78   /**
  79    * The default set of return properties.
  80    *
  81    * @var array
  82    */
  83   public static $_defaultReturnProperties = NULL;
  84
  85   /**
  86    * The default set of hier return properties.
  87    *
  88    * @var array
  89    */
  90   public static $_defaultHierReturnProperties;
  91
  92   /**
  93    * The set of input params.
  94    *
  95    * @var array
  96    */
  97   public $_params;
  98
  99   public $_cfIDs;
 100
 101   public $_paramLookup;
 102
 103   public $_sort;
 104
 105   /**
 106    * The set of output params
 107    *
 108    * @var array
 109    */
 110   public $_returnProperties;
 111
 112   /**
 113    * The select clause
 114    *
 115    * @var array
 116    */
 117   public $_select;
 118
 119   /**
 120    * The name of the elements that are in the select clause
 121    * used to extract the values.
 122    *
 123    * @var array
 124    */
 125   public $_element;
 126
 127   /**
 128    * The tables involved in the query.
 129    *
 130    * @var array
 131    */
 132   public $_tables;
 133
 134   /**
 135    * The table involved in the where clause.
 136    *
 137    * @var array
 138    */
 139   public $_whereTables;
 140
 141   /**
 142    * Array of WHERE clause components.
 143    *
 144    * @var array
 145    */
 146   public $_where;
 147
 148   /**
 149    * The WHERE clause as a string.
 150    *
 151    * @var string
 152    */
 153   public $_whereClause;
 154
 155   /**
 156    * Additional WHERE clause for permissions.
 157    *
 158    * @var string
 159    */
 160   public $_permissionWhereClause;
 161
 162   /**
 163    * The from string
 164    *
 165    * @var string
 166    */
 167   public $_fromClause;
 168
 169   /**
 170    * Additional permission from clause
 171    *
 172    * @var string
 173    */
 174   public $_permissionFromClause;
 175
 176   /**
 177    * The from clause for the simple select and alphabetical
 178    * select
 179    *
 180    * @var string
 181    */
 182   public $_simpleFromClause;
 183
 184   /**
 185    * The having values
 186    *
 187    * @var string
 188    */
 189   public $_having;
 190
 191   /**
 192    * The english language version of the query
 193    *
 194    * @var array
 195    */
 196   public $_qill;
 197
 198   /**
 199    * All the fields that could potentially be involved in
 200    * this query
 201    *
 202    * @var array
 203    */
 204   public $_fields;
 205
 206   /**
 207    * Fields hacked for legacy reasons.
 208    *
 209    * Generally where a field has a option group defining it's options we add them to
 210    * the fields array as pseudofields - eg for gender we would add the key 'gender' to fields
 211    * using CRM_Core_DAO::appendPseudoConstantsToFields($fields);
 212    *
 213    * The rendered results would hold an id in the gender_id field and the label in the pseudo 'Gender'
 214    * field. The heading for the pseudofield would come form the the option group name & for the id field
 215    * from the xml.
 216    *
 217    * These fields are handled in a more legacy way - ie overwriting 'gender_id' with the label on output
 218    * via the convertToPseudoNames function. Ideally we would convert them but they would then need to be fixed
 219    * in some other places & there are also some issues around the name (ie. Gender currently has the label in the
 220    * schema 'Gender' so adding a second 'Gender' field to search builder & export would be confusing and the standard is
 221    * not fully agreed here.
 222    *
 223    * @var array
 224    */
 225   protected $legacyHackedFields = [
 226     'gender_id' => 'gender',
 227     'prefix_id' => 'individual_prefix',
 228     'suffix_id' => 'individual_suffix',
 229     'communication_style_id' => 'communication_style',
 230   ];
 231
 232   /**
 233    * The cache to translate the option values into labels.
 234    *
 235    * @var array
 236    */
 237   public $_options;
 238
 239   /**
 240    * Are we in search mode.
 241    *
 242    * @var bool
 243    */
 244   public $_search = TRUE;
 245
 246   /**
 247    * Should we skip permission checking.
 248    *
 249    * @var bool
 250    */
 251   public $_skipPermission = FALSE;
 252
 253   /**
 254    * Should we skip adding of delete clause.
 255    *
 256    * @var bool
 257    */
 258   public $_skipDeleteClause = FALSE;
 259
 260   /**
 261    * Are we in strict mode (use equality over LIKE)
 262    *
 263    * @var bool
 264    */
 265   public $_strict = FALSE;
 266
 267   /**
 268    * What operator to use to group the clauses.
 269    *
 270    * @var string
 271    */
 272   public $_operator = 'AND';
 273
 274   public $_mode = 1;
 275
 276   /**
 277    * Should we only search on primary location.
 278    *
 279    * @var bool
 280    */
 281   public $_primaryLocation = TRUE;
 282
 283   /**
 284    * Are contact ids part of the query.
 285    *
 286    * @var bool
 287    */
 288   public $_includeContactIds = FALSE;
 289
 290   /**
 291    * Should we use the smart group cache.
 292    *
 293    * @var bool
 294    */
 295   public $_smartGroupCache = TRUE;
 296
 297   /**
 298    * Should we display contacts with a specific relationship type.
 299    *
 300    * @var string
 301    */
 302   public $_displayRelationshipType = NULL;
 303
 304   /**
 305    * Reference to the query object for custom values.
 306    *
 307    * @var Object
 308    */
 309   public $_customQuery;
 310
 311   /**
 312    * Should we enable the distinct clause, used if we are including
 313    * more than one group
 314    *
 315    * @var bool
 316    */
 317   public $_useDistinct = FALSE;
 318
 319   /**
 320    * Should we just display one contact record
 321    * @var bool
 322    */
 323   public $_useGroupBy = FALSE;
 324
 325   /**
 326    * The relationship type direction
 327    *
 328    * @var array
 329    */
 330   public static $_relType;
 331
 332   /**
 333    * The activity role
 334    *
 335    * @var array
 336    */
 337   public static $_activityRole;
 338
 339   /**
 340    * Consider the component activity type
 341    * during activity search.
 342    *
 343    * @var array
 344    */
 345   public static $_considerCompActivities;
 346
 347   /**
 348    * Consider with contact activities only,
 349    * during activity search.
 350    *
 351    * @var array
 352    */
 353   public static $_withContactActivitiesOnly;
 354
 355   /**
 356    * Use distinct component clause for component searches
 357    *
 358    * @var string
 359    */
 360   public $_distinctComponentClause;
 361
 362   public $_rowCountClause;
 363
 364   /**
 365    * Use groupBy component clause for component searches
 366    *
 367    * @var string
 368    */
 369   public $_groupByComponentClause;
 370
 371   /**
 372    * Track open panes, useful in advance search
 373    *
 374    * @var array
 375    */
 376   public static $_openedPanes = [];
 377
 378   /**
 379    * For search builder - which custom fields are location-dependent
 380    * @var array
 381    */
 382   public $_locationSpecificCustomFields = [];
 383
 384   /**
 385    * The tables which have a dependency on location and/or address
 386    *
 387    * @var array
 388    */
 389   public static $_dependencies = [
 390     'civicrm_state_province' => 1,
 391     'civicrm_country' => 1,
 392     'civicrm_county' => 1,
 393     'civicrm_address' => 1,
 394     'civicrm_location_type' => 1,
 395   ];
 396
 397   /**
 398    * List of location specific fields.
 399    * @var array
 400    */
 401   public static $_locationSpecificFields = [
 402     'street_address',
 403     'street_number',
 404     'street_name',
 405     'street_unit',
 406     'supplemental_address_1',
 407     'supplemental_address_2',
 408     'supplemental_address_3',
 409     'city',
 410     'postal_code',
 411     'postal_code_suffix',
 412     'geo_code_1',
 413     'geo_code_2',
 414     'state_province',
 415     'country',
 416     'county',
 417     'phone',
 418     'email',
 419     'im',
 420     'address_name',
 421     'master_id',
 422   ];
 423
 424   /**
 425    * Remember if we handle either end of a number or date range
 426    * so we can skip the other
 427    * @var array
 428    */
 429   protected $_rangeCache = [];
 430   /**
 431    * Set to true when $this->relationship is run to avoid adding twice.
 432    *
 433    * @var bool
 434    */
 435   protected $_relationshipValuesAdded = FALSE;
 436
 437   /**
 438    * Set to the name of the temp table if one has been created.
 439    *
 440    * @var string
 441    */
 442   public static $_relationshipTempTable = NULL;
 443
 444   public $_pseudoConstantsSelect = [];
 445
 446   public $_groupUniqueKey = NULL;
 447   public $_groupKeys = [];
 448
 449   /**
 450    * Class constructor which also does all the work.
 451    *
 452    * @param array $params
 453    * @param array $returnProperties
 454    * @param array $fields
 455    * @param bool $includeContactIds
 456    * @param bool $strict
 457    * @param bool|int $mode - mode the search is operating on
 458    *
 459    * @param bool $skipPermission
 460    * @param bool $searchDescendentGroups
 461    * @param bool $smartGroupCache
 462    * @param null $displayRelationshipType
 463    * @param string $operator
 464    * @param string $apiEntity
 465    * @param bool|null $primaryLocationOnly
 466    */
 467   public function __construct(
 468     $params = NULL, $returnProperties = NULL, $fields = NULL,
 469     $includeContactIds = FALSE, $strict = FALSE, $mode = 1,
 470     $skipPermission = FALSE, $searchDescendentGroups = TRUE,
 471     $smartGroupCache = TRUE, $displayRelationshipType = NULL,
 472     $operator = 'AND',
 473     $apiEntity = NULL,
 474     $primaryLocationOnly = NULL
 475   ) {
 476     if ($primaryLocationOnly === NULL) {
 477       $primaryLocationOnly = Civi::settings()->get('searchPrimaryDetailsOnly');
 478     }
 479     $this->_primaryLocation = $primaryLocationOnly;
 480     $this->_params = &$params;
 481     if ($this->_params == NULL) {
 482       $this->_params = [];
 483     }
 484
 485     if ($returnProperties === self::NO_RETURN_PROPERTIES) {
 486       $this->_returnProperties = [];
 487     }
 488     elseif (empty($returnProperties)) {
 489       $this->_returnProperties = self::defaultReturnProperties($mode);
 490     }
 491     else {
 492       $this->_returnProperties = &$returnProperties;
 493     }
 494
 495     $this->_includeContactIds = $includeContactIds;
 496     $this->_strict = $strict;
 497     $this->_mode = $mode;
 498     $this->_skipPermission = $skipPermission;
 499     $this->_smartGroupCache = $smartGroupCache;
 500     $this->_displayRelationshipType = $displayRelationshipType;
 501     $this->setOperator($operator);
 502
 503     if ($fields) {
 504       $this->_fields = &$fields;
 505       $this->_search = FALSE;
 506       $this->_skipPermission = TRUE;
 507     }
 508     else {
 509       $this->_fields = CRM_Contact_BAO_Contact::exportableFields('All', FALSE, TRUE, TRUE, FALSE, !$skipPermission);
 510       //  The legacy hacked fields will output as a string rather than their underlying type.
 511       foreach (array_keys($this->legacyHackedFields) as $fieldName) {
 512         $this->_fields[$fieldName]['type'] = CRM_Utils_Type::T_STRING;
 513       }
 514
 515       $fields = CRM_Core_Component::getQueryFields(!$this->_skipPermission);
 516       unset($fields['note']);
 517       $this->_fields = array_merge($this->_fields, $fields);
 518
 519       // add activity fields
 520       $fields = CRM_Activity_BAO_Activity::exportableFields();
 521       $this->_fields = array_merge($this->_fields, $fields);
 522
 523       // add any fields provided by hook implementers
 524       $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
 525       $this->_fields = array_merge($this->_fields, $extFields);
 526     }
 527
 528     // basically do all the work once, and then reuse it
 529     $this->initialize($apiEntity);
 530   }
 531
 532   /**
 533    * Function which actually does all the work for the constructor.
 534    *
 535    * @param string $apiEntity
 536    *   The api entity being called.
 537    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 538    */
 539   public function initialize($apiEntity = NULL) {
 540     $this->_select = [];
 541     $this->_element = [];
 542     $this->_tables = [];
 543     $this->_whereTables = [];
 544     $this->_where = [];
 545     $this->_qill = [];
 546     $this->_options = [];
 547     $this->_cfIDs = [];
 548     $this->_paramLookup = [];
 549     $this->_having = [];
 550
 551     $this->_customQuery = NULL;
 552
 553     // reset cached static variables - CRM-5803
 554     self::$_activityRole = NULL;
 555     self::$_considerCompActivities = NULL;
 556     self::$_withContactActivitiesOnly = NULL;
 557
 558     $this->_select['contact_id'] = 'contact_a.id as contact_id';
 559     $this->_element['contact_id'] = 1;
 560     $this->_tables['civicrm_contact'] = 1;
 561
 562     if (!empty($this->_params)) {
 563       $this->buildParamsLookup();
 564     }
 565
 566     $this->_whereTables = $this->_tables;
 567
 568     $this->selectClause($apiEntity);
 569     $isForcePrimaryOnly = !empty($apiEntity);
 570     $this->_whereClause = $this->whereClause($isForcePrimaryOnly);
 571     if (array_key_exists('civicrm_contribution', $this->_whereTables)) {
 572       $component = 'contribution';
 573     }
 574     if (array_key_exists('civicrm_membership', $this->_whereTables)) {
 575       $component = 'membership';
 576     }
 577     if (isset($component) && !$this->_skipPermission) {
 578       // Unit test coverage in api_v3_FinancialTypeACLTest::testGetACLContribution.
 579       CRM_Financial_BAO_FinancialType::buildPermissionedClause($this->_whereClause, $component);
 580     }
 581
 582     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode, $apiEntity);
 583     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
 584
 585     $this->openedSearchPanes(TRUE);
 586   }
 587
 588   /**
 589    * Function for same purpose as convertFormValues.
 590    *
 591    * Like convert form values this function exists to pre-Process parameters from the form.
 592    *
 593    * It is unclear why they are different functions & likely relates to advances search
 594    * versus search builder.
 595    *
 596    * The direction we are going is having the form convert values to a standardised format &
 597    * moving away from weird & wonderful where clause switches.
 598    *
 599    * Fix and handle contact deletion nicely.
 600    *
 601    * this code is primarily for search builder use case where different clauses can specify if they want deleted.
 602    *
 603    * CRM-11971
 604    */
 605   public function buildParamsLookup() {
 606     $trashParamExists = FALSE;
 607     $paramByGroup = [];
 608     foreach ($this->_params as $k => $param) {
 609       if (!empty($param[0]) && $param[0] == 'contact_is_deleted') {
 610         $trashParamExists = TRUE;
 611       }
 612       if (!empty($param[3])) {
 613         $paramByGroup[$param[3]][$k] = $param;
 614       }
 615     }
 616
 617     if ($trashParamExists) {
 618       $this->_skipDeleteClause = TRUE;
 619
 620       //cycle through group sets and explicitly add trash param if not set
 621       foreach ($paramByGroup as $setID => $set) {
 622         if (
 623           !in_array(['contact_is_deleted', '=', '1', $setID, '0'], $this->_params) &&
 624           !in_array(['contact_is_deleted', '=', '0', $setID, '0'], $this->_params)
 625         ) {
 626           $this->_params[] = [
 627             'contact_is_deleted',
 628             '=',
 629             '0',
 630             $setID,
 631             '0',
 632           ];
 633         }
 634       }
 635     }
 636
 637     foreach ($this->_params as $value) {
 638       if (empty($value[0])) {
 639         continue;
 640       }
 641       $cfID = CRM_Core_BAO_CustomField::getKeyID(str_replace('_relative', '', $value[0]));
 642       if ($cfID) {
 643         if (!array_key_exists($cfID, $this->_cfIDs)) {
 644           $this->_cfIDs[$cfID] = [];
 645         }
 646         // Set wildcard value based on "and/or" selection
 647         foreach ($this->_params as $key => $param) {
 648           if ($param[0] == $value[0] . '_operator') {
 649             $value[4] = $param[2] == 'or';
 650             break;
 651           }
 652         }
 653         $this->_cfIDs[$cfID][] = $value;
 654       }
 655
 656       if (!array_key_exists($value[0], $this->_paramLookup)) {
 657         $this->_paramLookup[$value[0]] = [];
 658       }
 659       if ($value[0] !== 'group') {
 660         // Just trying to unravel how group interacts here! This whole function is weird.
 661         $this->_paramLookup[$value[0]][] = $value;
 662       }
 663     }
 664   }
 665
 666   /**
 667    * Some composite fields do not appear in the fields array hack to make them part of the query.
 668    *
 669    * @param $apiEntity
 670    *   The api entity being called.
 671    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 672    */
 673   public function addSpecialFields($apiEntity) {
 674     static $special = ['contact_type', 'contact_sub_type', 'sort_name', 'display_name'];
 675     // if get called via Contact.get API having address_id as return parameter
 676     if ($apiEntity == 'Contact') {
 677       $special[] = 'address_id';
 678     }
 679     foreach ($special as $name) {
 680       if (!empty($this->_returnProperties[$name])) {
 681         if ($name == 'address_id') {
 682           $this->_tables['civicrm_address'] = 1;
 683           $this->_select['address_id'] = 'civicrm_address.id as address_id';
 684           $this->_element['address_id'] = 1;
 685         }
 686         else {
 687           $this->_select[$name] = "contact_a.{$name} as $name";
 688           $this->_element[$name] = 1;
 689         }
 690       }
 691     }
 692   }
 693
 694   /**
 695    * Given a list of conditions in params and a list of desired
 696    * return Properties generate the required select and from
 697    * clauses. Note that since the where clause introduces new
 698    * tables, the initial attempt also retrieves all variables used
 699    * in the params list
 700    *
 701    * @param string $apiEntity
 702    *   The api entity being called.
 703    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
 704    */
 705   public function selectClause($apiEntity = NULL) {
 706
 707     // @todo Tidy up this. This arises because 1) we are ignoring the $mode & adding a new
 708     // param ($apiEntity) instead - presumably an oversight & 2 because
 709     // contact is not implemented as a component.
 710     $this->addSpecialFields($apiEntity);
 711
 712     foreach ($this->_fields as $name => $field) {
 713       // skip component fields
 714       // there are done by the alter query below
 715       // and need not be done on every field
 716       // @todo remove these & handle using metadata - only obscure fields
 717       // that are hack-added should need to be excluded from the main loop.
 718       if (
 719         (substr($name, 0, 12) == 'participant_') ||
 720         (substr($name, 0, 7) == 'pledge_') ||
 721         (substr($name, 0, 5) == 'case_')
 722       ) {
 723         continue;
 724       }
 725
 726       // redirect to activity select clause
 727       if (
 728         (substr($name, 0, 9) == 'activity_') ||
 729         ($name == 'parent_id')
 730       ) {
 731         CRM_Activity_BAO_Query::select($this);
 732       }
 733
 734       // if this is a hierarchical name, we ignore it
 735       $names = explode('-', $name);
 736       if (count($names) > 1 && isset($names[1]) && is_numeric($names[1])) {
 737         continue;
 738       }
 739
 740       // make an exception for special cases, to add the field in select clause
 741       $makeException = FALSE;
 742
 743       //special handling for groups/tags
 744       if (in_array($name, ['groups', 'tags', 'notes'])
 745         && isset($this->_returnProperties[substr($name, 0, -1)])
 746       ) {
 747         // @todo instead of setting make exception to get us into
 748         // an if clause that has handling for these fields buried with in it
 749         // move the handling to here.
 750         $makeException = TRUE;
 751       }
 752
 753       // since note has 3 different options we need special handling
 754       // note / note_subject / note_body
 755       if ($name == 'notes') {
 756         foreach (['note', 'note_subject', 'note_body'] as $noteField) {
 757           if (isset($this->_returnProperties[$noteField])) {
 758             $makeException = TRUE;
 759             break;
 760           }
 761         }
 762       }
 763
 764       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
 765       if (
 766         !empty($this->_paramLookup[$name])
 767         || !empty($this->_returnProperties[$name])
 768         || $this->pseudoConstantNameIsInReturnProperties($field, $name)
 769         || $makeException
 770       ) {
 771         if ($cfID) {
 772           // add to cfIDs array if not present
 773           if (!array_key_exists($cfID, $this->_cfIDs)) {
 774             $this->_cfIDs[$cfID] = [];
 775           }
 776         }
 777         elseif (isset($field['where'])) {
 778           list($tableName, $fieldName) = explode('.', $field['where'], 2);
 779           if (isset($tableName)) {
 780             if (CRM_Utils_Array::value($tableName, self::$_dependencies)) {
 781               $this->_tables['civicrm_address'] = 1;
 782               $this->_select['address_id'] = 'civicrm_address.id as address_id';
 783               $this->_element['address_id'] = 1;
 784             }
 785
 786             if ($tableName == 'im_provider' || $tableName == 'email_greeting' ||
 787               $tableName == 'postal_greeting' || $tableName == 'addressee'
 788             ) {
 789               if ($tableName == 'im_provider') {
 790                 CRM_Core_OptionValue::select($this);
 791               }
 792
 793               if (in_array($tableName,
 794                 ['email_greeting', 'postal_greeting', 'addressee'])) {
 795                 $this->_element["{$name}_id"] = 1;
 796                 $this->_select["{$name}_id"] = "contact_a.{$name}_id as {$name}_id";
 797                 $this->_pseudoConstantsSelect[$name] = ['pseudoField' => $tableName, 'idCol' => "{$name}_id"];
 798                 $this->_pseudoConstantsSelect[$name]['select'] = "{$name}.{$fieldName} as $name";
 799                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 800
 801                 if ($tableName == 'email_greeting') {
 802                   // @todo bad join.
 803                   $this->_pseudoConstantsSelect[$name]['join']
 804                     = " LEFT JOIN civicrm_option_group option_group_email_greeting ON (option_group_email_greeting.name = 'email_greeting')";
 805                   $this->_pseudoConstantsSelect[$name]['join'] .=
 806                     " LEFT JOIN civicrm_option_value email_greeting ON (contact_a.email_greeting_id = email_greeting.value AND option_group_email_greeting.id = email_greeting.option_group_id ) ";
 807                 }
 808                 elseif ($tableName == 'postal_greeting') {
 809                   // @todo bad join.
 810                   $this->_pseudoConstantsSelect[$name]['join']
 811                     = " LEFT JOIN civicrm_option_group option_group_postal_greeting ON (option_group_postal_greeting.name = 'postal_greeting')";
 812                   $this->_pseudoConstantsSelect[$name]['join'] .=
 813                     " LEFT JOIN civicrm_option_value postal_greeting ON (contact_a.postal_greeting_id = postal_greeting.value AND option_group_postal_greeting.id = postal_greeting.option_group_id ) ";
 814                 }
 815                 elseif ($tableName == 'addressee') {
 816                   // @todo bad join.
 817                   $this->_pseudoConstantsSelect[$name]['join']
 818                     = " LEFT JOIN civicrm_option_group option_group_addressee ON (option_group_addressee.name = 'addressee')";
 819                   $this->_pseudoConstantsSelect[$name]['join'] .=
 820                     " LEFT JOIN civicrm_option_value addressee ON (contact_a.addressee_id = addressee.value AND option_group_addressee.id = addressee.option_group_id ) ";
 821                 }
 822                 $this->_pseudoConstantsSelect[$name]['table'] = $tableName;
 823
 824                 //get display
 825                 $greetField = "{$name}_display";
 826                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 827                 $this->_element[$greetField] = 1;
 828                 //get custom
 829                 $greetField = "{$name}_custom";
 830                 $this->_select[$greetField] = "contact_a.{$greetField} as {$greetField}";
 831                 $this->_element[$greetField] = 1;
 832               }
 833             }
 834             else {
 835               if (!in_array($tableName, ['civicrm_state_province', 'civicrm_country', 'civicrm_county'])) {
 836                 $this->_tables[$tableName] = 1;
 837               }
 838
 839               // also get the id of the tableName
 840               $tName = substr($tableName, 8);
 841               if (in_array($tName, ['country', 'state_province', 'county'])) {
 842                 if ($tName == 'state_province') {
 843                   $this->_pseudoConstantsSelect['state_province_name'] = [
 844                     'pseudoField' => "{$tName}",
 845                     'idCol' => "{$tName}_id",
 846                     'bao' => 'CRM_Core_BAO_Address',
 847                     'table' => "civicrm_{$tName}",
 848                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 849                   ];
 850
 851                   $this->_pseudoConstantsSelect[$tName] = [
 852                     'pseudoField' => 'state_province_abbreviation',
 853                     'idCol' => "{$tName}_id",
 854                     'table' => "civicrm_{$tName}",
 855                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 856                   ];
 857                 }
 858                 else {
 859                   $this->_pseudoConstantsSelect[$name] = [
 860                     'pseudoField' => "{$tName}_id",
 861                     'idCol' => "{$tName}_id",
 862                     'bao' => 'CRM_Core_BAO_Address',
 863                     'table' => "civicrm_{$tName}",
 864                     'join' => " LEFT JOIN civicrm_{$tName} ON civicrm_address.{$tName}_id = civicrm_{$tName}.id ",
 865                   ];
 866                 }
 867
 868                 $this->_select["{$tName}_id"] = "civicrm_address.{$tName}_id as {$tName}_id";
 869                 $this->_element["{$tName}_id"] = 1;
 870               }
 871               elseif ($tName != 'contact') {
 872                 $this->_select["{$tName}_id"] = "{$tableName}.id as {$tName}_id";
 873                 $this->_element["{$tName}_id"] = 1;
 874               }
 875
 876               //special case for phone
 877               if ($name == 'phone') {
 878                 $this->_select['phone_type_id'] = "civicrm_phone.phone_type_id as phone_type_id";
 879                 $this->_element['phone_type_id'] = 1;
 880               }
 881
 882               // if IM then select provider_id also
 883               // to get "IM Service Provider" in a file to be exported, CRM-3140
 884               if ($name == 'im') {
 885                 $this->_select['provider_id'] = "civicrm_im.provider_id as provider_id";
 886                 $this->_element['provider_id'] = 1;
 887               }
 888
 889               if ($tName == 'contact' && $fieldName == 'organization_name') {
 890                 // special case, when current employer is set for Individual contact
 891                 $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', NULL, contact_a.organization_name ) as organization_name";
 892               }
 893               elseif ($tName == 'contact' && $fieldName === 'id') {
 894                 // Handled elsewhere, explicitly ignore. Possibly for all tables...
 895               }
 896               elseif (in_array($tName, ['country', 'county'])) {
 897                 $this->_pseudoConstantsSelect[$name]['select'] = "{$field['where']} as `$name`";
 898                 $this->_pseudoConstantsSelect[$name]['element'] = $name;
 899               }
 900               elseif ($tName == 'state_province') {
 901                 $this->_pseudoConstantsSelect[$tName]['select'] = "{$field['where']} as `$name`";
 902                 $this->_pseudoConstantsSelect[$tName]['element'] = $name;
 903               }
 904               elseif (strpos($name, 'contribution_soft_credit') !== FALSE) {
 905                 if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled($this->_params)) {
 906                   $this->_select[$name] = "{$field['where']} as `$name`";
 907                 }
 908               }
 909               elseif ($this->pseudoConstantNameIsInReturnProperties($field, $name)) {
 910                 $this->addPseudoconstantFieldToSelect($name);
 911               }
 912               else {
 913                 $this->_select[$name] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$name`");
 914               }
 915               if (!in_array($tName, ['state_province', 'country', 'county'])) {
 916                 $this->_element[$name] = 1;
 917               }
 918             }
 919           }
 920         }
 921         elseif ($name === 'tags') {
 922           //@todo move this handling outside the big IF & ditch $makeException
 923           $this->_useGroupBy = TRUE;
 924           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_tag.name)) as tags";
 925           $this->_element[$name] = 1;
 926           $this->_tables['civicrm_tag'] = 1;
 927           $this->_tables['civicrm_entity_tag'] = 1;
 928         }
 929         elseif ($name === 'groups') {
 930           //@todo move this handling outside the big IF & ditch $makeException
 931           $this->_useGroupBy = TRUE;
 932           // Duplicates will be created here but better to sort them out in php land.
 933           $this->_select[$name] = "
 934             CONCAT_WS(',',
 935             GROUP_CONCAT(DISTINCT IF(civicrm_group_contact.status = 'Added', civicrm_group_contact.group_id, '')),
 936             GROUP_CONCAT(DISTINCT civicrm_group_contact_cache.group_id)
 937           )
 938           as groups";
 939           $this->_element[$name] = 1;
 940           $this->_tables['civicrm_group_contact'] = 1;
 941           $this->_tables['civicrm_group_contact_cache'] = 1;
 942           $this->_pseudoConstantsSelect["{$name}"] = [
 943             'pseudoField' => "groups",
 944             'idCol' => "groups",
 945           ];
 946         }
 947         elseif ($name === 'notes') {
 948           //@todo move this handling outside the big IF & ditch $makeException
 949           // if note field is subject then return subject else body of the note
 950           $noteColumn = 'note';
 951           if (isset($noteField) && $noteField == 'note_subject') {
 952             $noteColumn = 'subject';
 953           }
 954
 955           $this->_useGroupBy = TRUE;
 956           $this->_select[$name] = "GROUP_CONCAT(DISTINCT(civicrm_note.$noteColumn)) as notes";
 957           $this->_element[$name] = 1;
 958           $this->_tables['civicrm_note'] = 1;
 959         }
 960         elseif ($name === 'current_employer') {
 961           $this->_select[$name] = "IF ( contact_a.contact_type = 'Individual', contact_a.organization_name, NULL ) as current_employer";
 962           $this->_element[$name] = 1;
 963         }
 964       }
 965
 966       if ($cfID && !empty($field['is_search_range'])) {
 967         // this is a custom field with range search enabled, so we better check for two/from values
 968         if (!empty($this->_paramLookup[$name . '_from'])) {
 969           if (!array_key_exists($cfID, $this->_cfIDs)) {
 970             $this->_cfIDs[$cfID] = [];
 971           }
 972           foreach ($this->_paramLookup[$name . '_from'] as $pID => $p) {
 973             // search in the cdID array for the same grouping
 974             $fnd = FALSE;
 975             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 976               if ($c[3] == $p[3]) {
 977                 $this->_cfIDs[$cfID][$cID][2]['from'] = $p[2];
 978                 $fnd = TRUE;
 979               }
 980             }
 981             if (!$fnd) {
 982               $p[2] = ['from' => $p[2]];
 983               $this->_cfIDs[$cfID][] = $p;
 984             }
 985           }
 986         }
 987         if (!empty($this->_paramLookup[$name . '_to'])) {
 988           if (!array_key_exists($cfID, $this->_cfIDs)) {
 989             $this->_cfIDs[$cfID] = [];
 990           }
 991           foreach ($this->_paramLookup[$name . '_to'] as $pID => $p) {
 992             // search in the cdID array for the same grouping
 993             $fnd = FALSE;
 994             foreach ($this->_cfIDs[$cfID] as $cID => $c) {
 995               if ($c[4] == $p[4]) {
 996                 $this->_cfIDs[$cfID][$cID][2]['to'] = $p[2];
 997                 $fnd = TRUE;
 998               }
 999             }
1000             if (!$fnd) {
1001               $p[2] = ['to' => $p[2]];
1002               $this->_cfIDs[$cfID][] = $p;
1003             }
1004           }
1005         }
1006       }
1007     }
1008
1009     // add location as hierarchical elements
1010     $this->addHierarchicalElements();
1011
1012     // add multiple field like website
1013     $this->addMultipleElements();
1014
1015     //fix for CRM-951
1016     CRM_Core_Component::alterQuery($this, 'select');
1017
1018     CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'select');
1019
1020     if (!empty($this->_cfIDs)) {
1021       // @todo This function is the select function but instead of running 'select' it
1022       // is running the whole query.
1023       $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
1024       $this->_customQuery->query();
1025       $this->_select = array_merge($this->_select, $this->_customQuery->_select);
1026       $this->_element = array_merge($this->_element, $this->_customQuery->_element);
1027       $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
1028       $this->_whereTables = array_merge($this->_whereTables, $this->_customQuery->_whereTables);
1029       $this->_options = $this->_customQuery->_options;
1030     }
1031   }
1032
1033   /**
1034    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1035    */
1036   public function addHierarchicalElements() {
1037     if (empty($this->_returnProperties['location'])) {
1038       return;
1039     }
1040     if (!is_array($this->_returnProperties['location'])) {
1041       return;
1042     }
1043
1044     $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
1045     $processed = [];
1046     $index = 0;
1047
1048     $addressCustomFields = CRM_Core_BAO_CustomField::getFieldsForImport('Address');
1049     $addressCustomFieldIds = [];
1050
1051     foreach ($this->_returnProperties['location'] as $name => $elements) {
1052       $lCond = self::getPrimaryCondition($name);
1053       $locationTypeId = is_numeric($name) ? NULL : array_search($name, $locationTypes);
1054
1055       if (!$lCond) {
1056         if ($locationTypeId === FALSE) {
1057           continue;
1058         }
1059         $lCond = "location_type_id = $locationTypeId";
1060         $this->_useDistinct = TRUE;
1061
1062         //commented for CRM-3256
1063         $this->_useGroupBy = TRUE;
1064       }
1065
1066       $name = str_replace(' ', '_', $name);
1067       $tName = "$name-location_type";
1068       $ltName = "`$name-location_type`";
1069       $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1070       $this->_select["{$tName}"] = "`$tName`.name as `{$tName}`";
1071       $this->_element["{$tName}_id"] = 1;
1072       $this->_element["{$tName}"] = 1;
1073
1074       $locationTypeName = $tName;
1075       $locationTypeJoin = [];
1076
1077       $addWhereCount = 0;
1078       foreach ($elements as $elementFullName => $dontCare) {
1079         $index++;
1080         $elementName = $elementCmpName = $elementFullName;
1081
1082         if (substr($elementCmpName, 0, 5) == 'phone') {
1083           $elementCmpName = 'phone';
1084         }
1085
1086         if (in_array($elementCmpName, array_keys($addressCustomFields))) {
1087           if ($cfID = CRM_Core_BAO_CustomField::getKeyID($elementCmpName)) {
1088             $addressCustomFieldIds[$cfID][$name] = 1;
1089           }
1090         }
1091         // add address table - doesn't matter if we do it mutliple times - it's the same data
1092         // @todo ditch the double processing of addressJoin
1093         if ((in_array($elementCmpName, self::$_locationSpecificFields) || !empty($addressCustomFieldIds))
1094           && !in_array($elementCmpName, ['email', 'phone', 'im', 'openid'])
1095         ) {
1096           list($aName, $addressJoin) = $this->addAddressTable($name, $lCond);
1097           $locationTypeJoin[$tName] = " ( $aName.location_type_id = $ltName.id ) ";
1098           $processed[$aName] = 1;
1099         }
1100
1101         $cond = $elementType = '';
1102         if (strpos($elementName, '-') !== FALSE) {
1103           // this is either phone, email or IM
1104           list($elementName, $elementType) = explode('-', $elementName);
1105
1106           if (($elementName != 'phone') && ($elementName != 'im')) {
1107             $cond = self::getPrimaryCondition($elementType);
1108           }
1109           // CRM-13011 : If location type is primary, do not restrict search to the phone
1110           // type id - we want the primary phone, regardless of what type it is.
1111           // Otherwise, restrict to the specified phone type for the given field.
1112           if ((!$cond) && ($elementName == 'phone')) {
1113             $cond = "phone_type_id = '$elementType'";
1114           }
1115           elseif ((!$cond) && ($elementName == 'im')) {
1116             // IM service provider id, CRM-3140
1117             $cond = "provider_id = '$elementType'";
1118           }
1119           $elementType = '-' . $elementType;
1120         }
1121
1122         $field = CRM_Utils_Array::value($elementName, $this->_fields);
1123
1124         // hack for profile, add location id
1125         if (!$field) {
1126           if ($elementType &&
1127             // fix for CRM-882( to handle phone types )
1128             !is_numeric($elementType)
1129           ) {
1130             if (is_numeric($name)) {
1131               $field = CRM_Utils_Array::value($elementName . "-Primary$elementType", $this->_fields);
1132             }
1133             else {
1134               $field = CRM_Utils_Array::value($elementName . "-$locationTypeId$elementType", $this->_fields);
1135             }
1136           }
1137           elseif (is_numeric($name)) {
1138             //this for phone type to work
1139             if (in_array($elementName, ['phone', 'phone_ext'])) {
1140               $field = CRM_Utils_Array::value($elementName . "-Primary" . $elementType, $this->_fields);
1141             }
1142             else {
1143               $field = CRM_Utils_Array::value($elementName . "-Primary", $this->_fields);
1144             }
1145           }
1146           else {
1147             //this is for phone type to work for profile edit
1148             if (in_array($elementName, ['phone', 'phone_ext'])) {
1149               $field = CRM_Utils_Array::value($elementName . "-$locationTypeId$elementType", $this->_fields);
1150             }
1151             else {
1152               $field = CRM_Utils_Array::value($elementName . "-$locationTypeId", $this->_fields);
1153             }
1154           }
1155         }
1156
1157         // Check if there is a value, if so also add to where Clause
1158         $addWhere = FALSE;
1159         if ($this->_params) {
1160           $nm = $elementName;
1161           if (isset($locationTypeId)) {
1162             $nm .= "-$locationTypeId";
1163           }
1164           if (!is_numeric($elementType)) {
1165             $nm .= "$elementType";
1166           }
1167
1168           foreach ($this->_params as $id => $values) {
1169             if ((is_array($values) && $values[0] == $nm) ||
1170               (in_array($elementName, ['phone', 'im'])
1171                 && (strpos($values[0], $nm) !== FALSE)
1172               )
1173             ) {
1174               $addWhere = TRUE;
1175               $addWhereCount++;
1176               break;
1177             }
1178           }
1179         }
1180
1181         if ($field && isset($field['where'])) {
1182           list($tableName, $fieldName) = explode('.', $field['where'], 2);
1183           $pf = substr($tableName, 8);
1184           $tName = $name . '-' . $pf . $elementType;
1185           if (isset($tableName)) {
1186             if ($tableName == 'civicrm_state_province' || $tableName == 'civicrm_country' || $tableName == 'civicrm_county') {
1187               $this->_select["{$tName}_id"] = "{$aName}.{$pf}_id as `{$tName}_id`";
1188             }
1189             else {
1190               $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1191             }
1192
1193             $this->_element["{$tName}_id"] = 1;
1194             if (substr($tName, -15) == '-state_province') {
1195               // FIXME: hack to fix CRM-1900
1196               $a = Civi::settings()->get('address_format');
1197
1198               if (substr_count($a, 'state_province_name') > 0) {
1199                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1200                   'pseudoField' => "{$pf}_id",
1201                   'idCol' => "{$tName}_id",
1202                   'bao' => 'CRM_Core_BAO_Address',
1203                 ];
1204                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.name as `{$name}-{$elementFullName}`";
1205               }
1206               else {
1207                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1208                   'pseudoField' => 'state_province_abbreviation',
1209                   'idCol' => "{$tName}_id",
1210                 ];
1211                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.abbreviation as `{$name}-{$elementFullName}`";
1212               }
1213             }
1214             else {
1215               if (substr($elementFullName, 0, 2) == 'im') {
1216                 $provider = "{$name}-{$elementFullName}-provider_id";
1217                 $this->_select[$provider] = "`$tName`.provider_id as `{$name}-{$elementFullName}-provider_id`";
1218                 $this->_element[$provider] = 1;
1219               }
1220               if ($pf == 'country' || $pf == 'county') {
1221                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"] = [
1222                   'pseudoField' => "{$pf}_id",
1223                   'idCol' => "{$tName}_id",
1224                   'bao' => 'CRM_Core_BAO_Address',
1225                 ];
1226                 $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['select'] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1227               }
1228               else {
1229                 $this->_select["{$name}-{$elementFullName}"] = "`$tName`.$fieldName as `{$name}-{$elementFullName}`";
1230               }
1231             }
1232
1233             if (in_array($pf, ['state_province', 'country', 'county'])) {
1234               $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['element'] = "{$name}-{$elementFullName}";
1235             }
1236             else {
1237               $this->_element["{$name}-{$elementFullName}"] = 1;
1238             }
1239
1240             if (empty($processed["`$tName`"])) {
1241               $processed["`$tName`"] = 1;
1242               $newName = $tableName . '_' . $index;
1243               switch ($tableName) {
1244                 case 'civicrm_phone':
1245                 case 'civicrm_email':
1246                 case 'civicrm_im':
1247                 case 'civicrm_openid':
1248
1249                   $this->_tables[$tName] = "\nLEFT JOIN $tableName `$tName` ON contact_a.id = `$tName`.contact_id";
1250                   if ($tableName != 'civicrm_phone') {
1251                     $this->_tables[$tName] .= " AND `$tName`.$lCond";
1252                   }
1253                   elseif (is_numeric($name)) {
1254                     $this->_select[$tName] = "IF (`$tName`.is_primary = $name, `$tName`.phone, NULL) as `$tName`";
1255                   }
1256
1257                   // this special case to add phone type
1258                   if ($cond) {
1259                     $phoneTypeCondition = " AND `$tName`.$cond ";
1260                     //gross hack to pickup corrupted data also, CRM-7603
1261                     if (strpos($cond, 'phone_type_id') !== FALSE) {
1262                       $phoneTypeCondition = " AND ( `$tName`.$cond OR `$tName`.phone_type_id IS NULL ) ";
1263                       if (!empty($lCond)) {
1264                         $phoneTypeCondition .= " AND ( `$tName`.$lCond ) ";
1265                       }
1266                     }
1267                     $this->_tables[$tName] .= $phoneTypeCondition;
1268                   }
1269
1270                   //build locationType join
1271                   $locationTypeJoin[$tName] = " ( `$tName`.location_type_id = $ltName.id )";
1272
1273                   if ($addWhere) {
1274                     $this->_whereTables[$tName] = $this->_tables[$tName];
1275                   }
1276                   break;
1277
1278                 case 'civicrm_state_province':
1279                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $tName;
1280                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1281                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.state_province_id";
1282                   if ($addWhere) {
1283                     $this->_whereTables["{$name}-address"] = $addressJoin;
1284                   }
1285                   break;
1286
1287                 case 'civicrm_country':
1288                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1289                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1290                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.country_id";
1291                   if ($addWhere) {
1292                     $this->_whereTables["{$name}-address"] = $addressJoin;
1293                   }
1294                   break;
1295
1296                 case 'civicrm_county':
1297                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['table'] = $newName;
1298                   $this->_pseudoConstantsSelect["{$name}-{$elementFullName}"]['join']
1299                     = "\nLEFT JOIN $tableName `$tName` ON `$tName`.id = $aName.county_id";
1300                   if ($addWhere) {
1301                     $this->_whereTables["{$name}-address"] = $addressJoin;
1302                   }
1303                   break;
1304
1305                 default:
1306                   if ($addWhere) {
1307                     $this->_whereTables["{$name}-address"] = $addressJoin;
1308                   }
1309                   break;
1310               }
1311             }
1312           }
1313         }
1314       }
1315
1316       // add location type  join
1317       $ltypeJoin = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locationTypeJoin) . " )";
1318       $this->_tables[$locationTypeName] = $ltypeJoin;
1319
1320       // table should be present in $this->_whereTables,
1321       // to add its condition in location type join, CRM-3939.
1322       if ($addWhereCount) {
1323         $locClause = [];
1324         foreach ($this->_whereTables as $tableName => $clause) {
1325           if (!empty($locationTypeJoin[$tableName])) {
1326             $locClause[] = $locationTypeJoin[$tableName];
1327           }
1328         }
1329
1330         if (!empty($locClause)) {
1331           $this->_whereTables[$locationTypeName] = "\nLEFT JOIN civicrm_location_type $ltName ON ( " . implode('OR', $locClause) . " )";
1332         }
1333       }
1334     }
1335
1336     if (!empty($addressCustomFieldIds)) {
1337       $customQuery = new CRM_Core_BAO_CustomQuery($addressCustomFieldIds);
1338       foreach ($addressCustomFieldIds as $cfID => $locTypeName) {
1339         foreach ($locTypeName as $name => $dnc) {
1340           $this->_locationSpecificCustomFields[$cfID] = [$name, array_search($name, $locationTypes)];
1341           $fieldName = "$name-custom_{$cfID}";
1342           $tName = "$name-address-custom-{$cfID}";
1343           $aName = "`$name-address-custom-{$cfID}`";
1344           $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1345           $this->_element["{$tName}_id"] = 1;
1346           $this->_select[$fieldName] = "`$tName`.{$customQuery->_fields[$cfID]['column_name']} as `{$fieldName}`";
1347           $this->_element[$fieldName] = 1;
1348           $this->_tables[$tName] = "\nLEFT JOIN {$customQuery->_fields[$cfID]['table_name']} $aName ON ($aName.entity_id = `$name-address`.id)";
1349         }
1350       }
1351     }
1352   }
1353
1354   /**
1355    * If the return Properties are set in a hierarchy, traverse the hierarchy to get the return values.
1356    */
1357   public function addMultipleElements() {
1358     if (empty($this->_returnProperties['website'])) {
1359       return;
1360     }
1361     if (!is_array($this->_returnProperties['website'])) {
1362       return;
1363     }
1364
1365     foreach ($this->_returnProperties['website'] as $key => $elements) {
1366       foreach ($elements as $elementFullName => $dontCare) {
1367         $tName = "website-{$key}-{$elementFullName}";
1368         $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
1369         $this->_select["{$tName}"] = "`$tName`.url as `{$tName}`";
1370         $this->_element["{$tName}_id"] = 1;
1371         $this->_element["{$tName}"] = 1;
1372
1373         $type = "website-{$key}-website_type_id";
1374         $this->_select[$type] = "`$tName`.website_type_id as `{$type}`";
1375         $this->_element[$type] = 1;
1376         $this->_tables[$tName] = "\nLEFT JOIN civicrm_website `$tName` ON (`$tName`.contact_id = contact_a.id AND `$tName`.website_type_id = $key )";
1377       }
1378     }
1379   }
1380
1381   /**
1382    * Generate the query based on what type of query we need.
1383    *
1384    * @param bool $count
1385    * @param bool $sortByChar
1386    * @param bool $groupContacts
1387    * @param bool $onlyDeleted
1388    *
1389    * @return array
1390    *   sql query parts as an array
1391    */
1392   public function query($count = FALSE, $sortByChar = FALSE, $groupContacts = FALSE, $onlyDeleted = FALSE) {
1393     // build permission clause
1394     $this->generatePermissionClause($onlyDeleted, $count);
1395
1396     if ($count) {
1397       if (isset($this->_rowCountClause)) {
1398         $select = "SELECT {$this->_rowCountClause}";
1399       }
1400       elseif (isset($this->_distinctComponentClause)) {
1401         // we add distinct to get the right count for components
1402         // for the more complex result set, we use GROUP BY the same id
1403         // CRM-9630
1404         $select = "SELECT count( DISTINCT {$this->_distinctComponentClause} ) as rowCount";
1405       }
1406       else {
1407         $select = 'SELECT count(DISTINCT contact_a.id) as rowCount';
1408       }
1409       $from = $this->_simpleFromClause;
1410       if ($this->_useDistinct) {
1411         $this->_useGroupBy = TRUE;
1412       }
1413     }
1414     elseif ($sortByChar) {
1415       // @fixme add the deprecated warning back in (it breaks CRM_Contact_SelectorTest::testSelectorQuery)
1416       // CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
1417       $select = 'SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name';
1418       $from = $this->_simpleFromClause;
1419     }
1420     elseif ($groupContacts) {
1421       $select = 'SELECT contact_a.id as id';
1422       if ($this->_useDistinct) {
1423         $this->_useGroupBy = TRUE;
1424       }
1425       $from = $this->_simpleFromClause;
1426     }
1427     else {
1428       if (!empty($this->_paramLookup['group'])) {
1429
1430         list($name, $op, $value, $grouping, $wildcard) = $this->_paramLookup['group'][0];
1431
1432         if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
1433           $this->_paramLookup['group'][0][1] = key($value);
1434         }
1435
1436         // Presumably the lines below come into manage groups screen.
1437         // make sure there is only one element
1438         // this is used when we are running under smog and need to know
1439         // how the contact was added (CRM-1203)
1440         $groups = (array) CRM_Utils_Array::value($this->_paramLookup['group'][0][1], $this->_paramLookup['group'][0][2], $this->_paramLookup['group'][0][2]);
1441         if ((count($this->_paramLookup['group']) == 1) &&
1442           (count($groups) == 1)
1443         ) {
1444           $groupId = $groups[0];
1445
1446           //check if group is saved search
1447           $group = new CRM_Contact_BAO_Group();
1448           $group->id = $groupId;
1449           $group->find(TRUE);
1450
1451           if (!isset($group->saved_search_id)) {
1452             $tbName = "civicrm_group_contact";
1453             // CRM-17254 don't retrieve extra fields if contact_id is specifically requested
1454             // as this will add load to an intentionally light query.
1455             // ideally this code would be removed as it appears to be to support CRM-1203
1456             // and passing in the required returnProperties from the url would
1457             // make more sense that globally applying the requirements of one form.
1458             if (($this->_returnProperties != ['contact_id'])) {
1459               $this->_select['group_contact_id'] = "$tbName.id as group_contact_id";
1460               $this->_element['group_contact_id'] = 1;
1461               $this->_select['status'] = "$tbName.status as status";
1462               $this->_element['status'] = 1;
1463             }
1464           }
1465         }
1466         $this->_useGroupBy = TRUE;
1467       }
1468       if ($this->_useDistinct && !isset($this->_distinctComponentClause)) {
1469         if (!($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY)) {
1470           // CRM-5954
1471           $this->_select['contact_id'] = 'contact_a.id as contact_id';
1472           $this->_useDistinct = FALSE;
1473           $this->_useGroupBy = TRUE;
1474         }
1475       }
1476
1477       $select = $this->getSelect();
1478       $from = $this->_fromClause;
1479     }
1480
1481     $where = '';
1482     if (!empty($this->_whereClause)) {
1483       $where = "WHERE {$this->_whereClause}";
1484     }
1485
1486     if (!empty($this->_permissionWhereClause) && empty($this->_displayRelationshipType)) {
1487       if (empty($where)) {
1488         $where = "WHERE $this->_permissionWhereClause";
1489       }
1490       else {
1491         $where = "$where AND $this->_permissionWhereClause";
1492       }
1493     }
1494
1495     $having = '';
1496     if (!empty($this->_having)) {
1497       foreach ($this->_having as $havingSets) {
1498         foreach ($havingSets as $havingSet) {
1499           $havingValue[] = $havingSet;
1500         }
1501       }
1502       $having = ' HAVING ' . implode(' AND ', $havingValue);
1503     }
1504
1505     // if we are doing a transform, do it here
1506     // use the $from, $where and $having to get the contact ID
1507     if ($this->_displayRelationshipType) {
1508       $this->filterRelatedContacts($from, $where, $having);
1509     }
1510
1511     return [$select, $from, $where, $having];
1512   }
1513
1514   /**
1515    * Get where values from the parameters.
1516    *
1517    * @param string $name
1518    * @param mixed $grouping
1519    *
1520    * @return mixed
1521    */
1522   public function getWhereValues($name, $grouping) {
1523     $result = NULL;
1524     foreach ($this->_params as $values) {
1525       if ($values[0] == $name && $values[3] == $grouping) {
1526         return $values;
1527       }
1528     }
1529
1530     return $result;
1531   }
1532
1533   /**
1534    * Fix date values.
1535    *
1536    * @param bool $relative
1537    * @param string $from
1538    * @param string $to
1539    */
1540   public static function fixDateValues($relative, &$from, &$to) {
1541     if ($relative) {
1542       list($from, $to) = CRM_Utils_Date::getFromTo($relative, $from, $to);
1543     }
1544   }
1545
1546   /**
1547    * Convert values from form-appropriate to query-object appropriate.
1548    *
1549    * The query object is increasingly supporting the sql-filter syntax which is the most flexible syntax.
1550    * So, ideally we would convert all fields to look like
1551    *  array(
1552    *   0 => $fieldName
1553    *   // Set the operator for legacy reasons, but it is ignored
1554    *   1 =>  '='
1555    *   // array in sql filter syntax
1556    *   2 => array('BETWEEN' => array(1,60),
1557    *   3 => null
1558    *   4 => null
1559    *  );
1560    *
1561    * There are some examples of the syntax in
1562    * https://github.com/civicrm/civicrm-core/tree/master/api/v3/examples/Relationship
1563    *
1564    * More notes at CRM_Core_DAO::createSQLFilter
1565    *
1566    * and a list of supported operators in CRM_Core_DAO
1567    *
1568    * @param array $formValues
1569    * @param int $wildcard
1570    * @param bool $useEquals
1571    *
1572    * @param string $apiEntity
1573    *
1574    * @param array $entityReferenceFields
1575    *   Field names of any entity reference fields (which will need reformatting to IN syntax).
1576    *
1577    * @return array
1578    */
1579   public static function convertFormValues(&$formValues, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL,
1580     $entityReferenceFields = []) {
1581     $params = [];
1582     if (empty($formValues)) {
1583       return $params;
1584     }
1585
1586     self::filterCountryFromValuesIfStateExists($formValues);
1587     // We shouldn't have to whitelist fields to not hack but here we are, for now.
1588     $nonLegacyDateFields = [
1589       'participant_register_date_relative',
1590       'receive_date_relative',
1591       'pledge_end_date_relative',
1592       'pledge_create_date_relative',
1593       'pledge_start_date_relative',
1594       'pledge_payment_scheduled_date_relative',
1595       'membership_join_date_relative',
1596       'membership_start_date_relative',
1597       'membership_end_date_relative',
1598       'case_start_date_relative',
1599       'case_end_date_relative',
1600     ];
1601     // Handle relative dates first
1602     foreach (array_keys($formValues) as $id) {
1603       if (
1604         !in_array($id, $nonLegacyDateFields) && (
1605         preg_match('/_date_relative$/', $id) ||
1606         $id == 'event_relative' ||
1607         $id == 'case_from_relative' ||
1608         $id == 'case_to_relative')
1609       ) {
1610         if ($id == 'event_relative') {
1611           $fromRange = 'event_start_date_low';
1612           $toRange = 'event_end_date_high';
1613         }
1614         elseif ($id == 'case_from_relative') {
1615           $fromRange = 'case_from_start_date_low';
1616           $toRange = 'case_from_start_date_high';
1617         }
1618         elseif ($id == 'case_to_relative') {
1619           $fromRange = 'case_to_end_date_low';
1620           $toRange = 'case_to_end_date_high';
1621         }
1622         else {
1623           $dateComponent = explode('_date_relative', $id);
1624           $fromRange = "{$dateComponent[0]}_date_low";
1625           $toRange = "{$dateComponent[0]}_date_high";
1626         }
1627
1628         if (array_key_exists($fromRange, $formValues) && array_key_exists($toRange, $formValues)) {
1629           CRM_Contact_BAO_Query::fixDateValues($formValues[$id], $formValues[$fromRange], $formValues[$toRange]);
1630         }
1631       }
1632     }
1633
1634     foreach ($formValues as $id => $values) {
1635       if (self::isAlreadyProcessedForQueryFormat($values)) {
1636         $params[] = $values;
1637         continue;
1638       }
1639
1640       self::legacyConvertFormValues($id, $values);
1641
1642       // The form uses 1 field to represent two db fields
1643       if ($id == 'contact_type' && $values && (!is_array($values) || !array_intersect(array_keys($values), CRM_Core_DAO::acceptedSQLOperators()))) {
1644         $contactType = [];
1645         $subType = [];
1646         foreach ((array) $values as $key => $type) {
1647           $types = explode('__', is_numeric($type) ? $key : $type, 2);
1648           $contactType[$types[0]] = $types[0];
1649           // Add sub-type if specified
1650           if (!empty($types[1])) {
1651             $subType[$types[1]] = $types[1];
1652           }
1653         }
1654         $params[] = ['contact_type', 'IN', $contactType, 0, 0];
1655         if ($subType) {
1656           $params[] = ['contact_sub_type', 'IN', $subType, 0, 0];
1657         }
1658       }
1659       elseif ($id == 'privacy') {
1660         if (is_array($formValues['privacy'])) {
1661           $op = !empty($formValues['privacy']['do_not_toggle']) ? '=' : '!=';
1662           foreach ($formValues['privacy'] as $key => $value) {
1663             if ($value) {
1664               $params[] = [$key, $op, $value, 0, 0];
1665             }
1666           }
1667         }
1668       }
1669       elseif ($id == 'email_on_hold') {
1670         if ($onHoldValue = CRM_Utils_Array::value('email_on_hold', $formValues)) {
1671           // onHoldValue should be 0 or 1 or an array. Some legacy groups may hold ''
1672           // so in 5.11 we have an extra if that should become redundant over time.
1673           // https://lab.civicrm.org/dev/core/issues/745
1674           // @todo this renaming of email_on_hold to on_hold needs revisiting
1675           // it preceeds recent changes but causes the default not to reload.
1676           $onHoldValue = array_filter((array) $onHoldValue, 'is_numeric');
1677           if (!empty($onHoldValue)) {
1678             $params[] = ['on_hold', 'IN', $onHoldValue, 0, 0];
1679           }
1680         }
1681       }
1682       elseif (substr($id, 0, 7) == 'custom_'
1683         &&  (
1684           substr($id, -5, 5) == '_from'
1685           || substr($id, -3, 3) == '_to'
1686         )
1687       ) {
1688         self::convertCustomRelativeFields($formValues, $params, $values, $id);
1689       }
1690       elseif (
1691         !in_array($id, $nonLegacyDateFields) && (
1692           preg_match('/_date_relative$/', $id) ||
1693           $id == 'event_relative' ||
1694           $id == 'case_from_relative' ||
1695           $id == 'case_to_relative')
1696       ) {
1697         // Already handled in previous loop
1698         continue;
1699       }
1700       elseif (in_array($id, $entityReferenceFields) && !empty($values) && is_string($values) && (strpos($values, ',') !=
1701         FALSE)) {
1702         $params[] = [$id, 'IN', explode(',', $values), 0, 0];
1703       }
1704       else {
1705         $values = CRM_Contact_BAO_Query::fixWhereValues($id, $values, $wildcard, $useEquals, $apiEntity);
1706
1707         if (!$values) {
1708           continue;
1709         }
1710         $params[] = $values;
1711       }
1712     }
1713     return $params;
1714   }
1715
1716   /**
1717    * Function to support legacy format for groups and tags.
1718    *
1719    * @param string $id
1720    * @param array|int $values
1721    *
1722    */
1723   public static function legacyConvertFormValues($id, &$values) {
1724     $legacyElements = [
1725       'group',
1726       'tag',
1727       'contact_tags',
1728       'contact_type',
1729       'membership_type_id',
1730       'membership_status_id',
1731     ];
1732     if (in_array($id, $legacyElements) && is_array($values)) {
1733       // prior to 4.7, formValues for some attributes (e.g. group, tag) are stored in array(id1 => 1, id2 => 1),
1734       // as per the recent Search fixes $values need to be in standard array(id1, id2) format
1735       $values = CRM_Utils_Array::convertCheckboxFormatToArray($values);
1736     }
1737   }
1738
1739   /**
1740    * Fix values from query from/to something no-one cared enough to document.
1741    *
1742    * @param int $id
1743    * @param array $values
1744    * @param int $wildcard
1745    * @param bool $useEquals
1746    *
1747    * @param string $apiEntity
1748    *
1749    * @return array|null
1750    */
1751   public static function fixWhereValues($id, &$values, $wildcard = 0, $useEquals = FALSE, $apiEntity = NULL) {
1752     // skip a few search variables
1753     static $skipWhere = NULL;
1754     static $likeNames = NULL;
1755     $result = NULL;
1756
1757     // Change camelCase EntityName to lowercase with underscores
1758     $apiEntity = _civicrm_api_get_entity_name_from_camel($apiEntity);
1759
1760     // check if $value is in OK (Operator as Key) format as used by Get API
1761     if (CRM_Utils_System::isNull($values)) {
1762       return $result;
1763     }
1764
1765     if (!$skipWhere) {
1766       $skipWhere = [
1767         'task',
1768         'radio_ts',
1769         'uf_group_id',
1770         'component_mode',
1771         'qfKey',
1772         'operator',
1773         'display_relationship_type',
1774       ];
1775     }
1776
1777     if (in_array($id, $skipWhere) ||
1778       substr($id, 0, 4) == '_qf_' ||
1779       substr($id, 0, 7) == 'hidden_'
1780     ) {
1781       return $result;
1782     }
1783
1784     if ($apiEntity &&
1785       (substr($id, 0, strlen($apiEntity)) != $apiEntity) &&
1786       (substr($id, 0, 10) != 'financial_' && substr($id, 0, 8) != 'payment_') &&
1787       (substr($id, 0, 7) != 'custom_')
1788     ) {
1789       $id = $apiEntity . '_' . $id;
1790     }
1791
1792     if (!$likeNames) {
1793       $likeNames = ['sort_name', 'email', 'note', 'display_name'];
1794     }
1795
1796     // email comes in via advanced search
1797     // so use wildcard always
1798     if ($id == 'email') {
1799       $wildcard = 1;
1800     }
1801
1802     if (!$useEquals && in_array($id, $likeNames)) {
1803       $result = [$id, 'LIKE', $values, 0, 1];
1804     }
1805     elseif (is_string($values) && strpos($values, '%') !== FALSE) {
1806       $result = [$id, 'LIKE', $values, 0, 0];
1807     }
1808     elseif ($id == 'contact_type' ||
1809       (!empty($values) && is_array($values) && !in_array(key($values), CRM_Core_DAO::acceptedSQLOperators(), TRUE))
1810     ) {
1811       $result = [$id, 'IN', $values, 0, $wildcard];
1812     }
1813     else {
1814       $result = [$id, '=', $values, 0, $wildcard];
1815     }
1816
1817     return $result;
1818   }
1819
1820   /**
1821    * Get the where clause for a single field.
1822    *
1823    * @param array $values
1824    * @param bool $isForcePrimaryOnly
1825    *
1826    * @throws \CRM_Core_Exception
1827    */
1828   public function whereClauseSingle(&$values, $isForcePrimaryOnly = FALSE) {
1829     if ($this->isARelativeDateField($values[0])) {
1830       $this->buildRelativeDateQuery($values);
1831       return;
1832     }
1833     // @todo also handle _low, _high generically here with if ($query->buildDateRangeQuery($values)) {return}
1834
1835     // do not process custom fields or prefixed contact ids or component params
1836     if (CRM_Core_BAO_CustomField::getKeyID($values[0]) ||
1837       (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) ||
1838       (substr($values[0], 0, 13) == 'contribution_') ||
1839       (substr($values[0], 0, 6) == 'event_') ||
1840       (substr($values[0], 0, 12) == 'participant_') ||
1841       (substr($values[0], 0, 7) == 'member_') ||
1842       (substr($values[0], 0, 6) == 'grant_') ||
1843       (substr($values[0], 0, 7) == 'pledge_') ||
1844       (substr($values[0], 0, 5) == 'case_') ||
1845       (substr($values[0], 0, 10) == 'financial_') ||
1846       (substr($values[0], 0, 8) == 'payment_') ||
1847       (substr($values[0], 0, 11) == 'membership_')
1848     ) {
1849       return;
1850     }
1851
1852     // skip for hook injected fields / params
1853     $extFields = CRM_Contact_BAO_Query_Hook::singleton()->getFields();
1854     if (array_key_exists($values[0], $extFields)) {
1855       return;
1856     }
1857
1858     switch ($values[0]) {
1859       case 'deleted_contacts':
1860         $this->deletedContacts($values);
1861         return;
1862
1863       case 'contact_sub_type':
1864         $this->contactSubType($values);
1865         return;
1866
1867       case 'group':
1868       case 'group_type':
1869         $this->group($values);
1870         return;
1871
1872       // case tag comes from find contacts
1873       case 'tag_search':
1874         $this->tagSearch($values);
1875         return;
1876
1877       case 'tag':
1878       case 'contact_tags':
1879         $this->tag($values);
1880         return;
1881
1882       case 'note':
1883       case 'note_body':
1884       case 'note_subject':
1885         $this->notes($values);
1886         return;
1887
1888       case 'uf_user':
1889         $this->ufUser($values);
1890         return;
1891
1892       case 'sort_name':
1893       case 'display_name':
1894         $this->sortName($values);
1895         return;
1896
1897       case 'addressee':
1898       case 'postal_greeting':
1899       case 'email_greeting':
1900         $this->greetings($values);
1901         return;
1902
1903       case 'email':
1904       case 'email_id':
1905         $this->email($values, $isForcePrimaryOnly);
1906         return;
1907
1908       case 'phone_numeric':
1909         $this->phone_numeric($values);
1910         return;
1911
1912       case 'phone_phone_type_id':
1913       case 'phone_location_type_id':
1914         $this->phone_option_group($values);
1915         return;
1916
1917       case 'street_address':
1918         $this->street_address($values);
1919         return;
1920
1921       case 'street_number':
1922         $this->street_number($values);
1923         return;
1924
1925       case 'sortByCharacter':
1926         $this->sortByCharacter($values);
1927         return;
1928
1929       case 'location_type':
1930         $this->locationType($values);
1931         return;
1932
1933       case 'county':
1934         $this->county($values);
1935         return;
1936
1937       case 'state_province':
1938       case 'state_province_id':
1939       case 'state_province_name':
1940         $this->stateProvince($values);
1941         return;
1942
1943       case 'country':
1944       case 'country_id':
1945         $this->country($values, FALSE);
1946         return;
1947
1948       case 'postal_code':
1949       case 'postal_code_low':
1950       case 'postal_code_high':
1951         $this->postalCode($values);
1952         return;
1953
1954       case 'activity_date':
1955       case 'activity_date_low':
1956       case 'activity_date_high':
1957       case 'activity_date_time_low':
1958       case 'activity_date_time_high':
1959       case 'activity_role':
1960       case 'activity_status_id':
1961       case 'activity_status':
1962       case 'activity_priority':
1963       case 'activity_priority_id':
1964       case 'followup_parent_id':
1965       case 'parent_id':
1966       case 'source_contact_id':
1967       case 'activity_text':
1968       case 'activity_option':
1969       case 'test_activities':
1970       case 'activity_type_id':
1971       case 'activity_type':
1972       case 'activity_survey_id':
1973       case 'activity_tags':
1974       case 'activity_taglist':
1975       case 'activity_test':
1976       case 'activity_campaign_id':
1977       case 'activity_engagement_level':
1978       case 'activity_id':
1979       case 'activity_result':
1980       case 'source_contact':
1981         CRM_Activity_BAO_Query::whereClauseSingle($values, $this);
1982         return;
1983
1984       case 'age_low':
1985       case 'age_high':
1986       case 'birth_date_low':
1987       case 'birth_date_high':
1988       case 'deceased_date_low':
1989       case 'deceased_date_high':
1990         $this->demographics($values);
1991         return;
1992
1993       case 'age_asof_date':
1994         // handled by demographics
1995         return;
1996
1997       case 'log_date_low':
1998       case 'log_date_high':
1999         $this->modifiedDates($values);
2000         return;
2001
2002       case 'changed_by':
2003         $this->changeLog($values);
2004         return;
2005
2006       case 'do_not_phone':
2007       case 'do_not_email':
2008       case 'do_not_mail':
2009       case 'do_not_sms':
2010       case 'do_not_trade':
2011       case 'is_opt_out':
2012         $this->privacy($values);
2013         return;
2014
2015       case 'privacy_options':
2016         $this->privacyOptions($values);
2017         return;
2018
2019       case 'privacy_operator':
2020       case 'privacy_toggle':
2021         // these are handled by privacy options
2022         return;
2023
2024       case 'preferred_communication_method':
2025         $this->preferredCommunication($values);
2026         return;
2027
2028       case 'relation_type_id':
2029       case 'relation_start_date_high':
2030       case 'relation_start_date_low':
2031       case 'relation_end_date_high':
2032       case 'relation_end_date_low':
2033       case 'relation_active_period_date_high':
2034       case 'relation_active_period_date_low':
2035       case 'relation_target_name':
2036       case 'relation_status':
2037       case 'relation_description':
2038       case 'relation_date_low':
2039       case 'relation_date_high':
2040         $this->relationship($values);
2041         $this->_relationshipValuesAdded = TRUE;
2042         return;
2043
2044       case 'task_status_id':
2045         $this->task($values);
2046         return;
2047
2048       case 'task_id':
2049         // since this case is handled with the above
2050         return;
2051
2052       case 'prox_distance':
2053         CRM_Contact_BAO_ProximityQuery::process($this, $values);
2054         return;
2055
2056       case 'prox_street_address':
2057       case 'prox_city':
2058       case 'prox_postal_code':
2059       case 'prox_state_province_id':
2060       case 'prox_country_id':
2061       case 'prox_geo_code_1':
2062       case 'prox_geo_code_2':
2063         // handled by the proximity_distance clause
2064         return;
2065
2066       default:
2067         $this->restWhere($values);
2068         return;
2069     }
2070   }
2071
2072   /**
2073    * Given a list of conditions in params generate the required where clause.
2074    *
2075    * @param bool $isForcePrimaryEmailOnly
2076    *
2077    * @return string
2078    * @throws \CRM_Core_Exception
2079    */
2080   public function whereClause($isForcePrimaryEmailOnly = NULL) {
2081     $this->_where[0] = [];
2082     $this->_qill[0] = [];
2083
2084     $this->includeContactIds();
2085     if (!empty($this->_params)) {
2086       foreach (array_keys($this->_params) as $id) {
2087         if (empty($this->_params[$id][0])) {
2088           continue;
2089         }
2090         // check for both id and contact_id
2091         if ($this->_params[$id][0] == 'id' || $this->_params[$id][0] == 'contact_id') {
2092           $this->_where[0][] = self::buildClause("contact_a.id", $this->_params[$id][1], $this->_params[$id][2]);
2093           $field = CRM_Utils_Array::value('id', $this->_fields);
2094           list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(
2095             'CRM_Contact_BAO_Contact',
2096             "contact_a.id",
2097             $this->_params[$id][2],
2098             $this->_params[$id][1]
2099           );
2100           $this->_qill[0][] = ts("%1 %2 %3", [
2101             1 => $field['title'],
2102             2 => $qillop,
2103             3 => $qillVal,
2104           ]);
2105         }
2106         else {
2107           $this->whereClauseSingle($this->_params[$id], $isForcePrimaryEmailOnly);
2108         }
2109       }
2110
2111       CRM_Core_Component::alterQuery($this, 'where');
2112
2113       CRM_Contact_BAO_Query_Hook::singleton()->alterSearchQuery($this, 'where');
2114     }
2115
2116     if ($this->_customQuery) {
2117       // Added following if condition to avoid the wrong value display for 'my account' / any UF info.
2118       // Hope it wont affect the other part of civicrm.. if it does please remove it.
2119       if (!empty($this->_customQuery->_where)) {
2120         $this->_where = CRM_Utils_Array::crmArrayMerge($this->_where, $this->_customQuery->_where);
2121       }
2122
2123       $this->_qill = CRM_Utils_Array::crmArrayMerge($this->_qill, $this->_customQuery->_qill);
2124     }
2125
2126     $clauses = [];
2127     $andClauses = [];
2128
2129     $validClauses = 0;
2130     if (!empty($this->_where)) {
2131       foreach ($this->_where as $grouping => $values) {
2132         if ($grouping > 0 && !empty($values)) {
2133           $clauses[$grouping] = ' ( ' . implode(" {$this->_operator} ", $values) . ' ) ';
2134           $validClauses++;
2135         }
2136       }
2137
2138       if (!empty($this->_where[0])) {
2139         $andClauses[] = ' ( ' . implode(" {$this->_operator} ", $this->_where[0]) . ' ) ';
2140       }
2141       if (!empty($clauses)) {
2142         $andClauses[] = ' ( ' . implode(' OR ', $clauses) . ' ) ';
2143       }
2144
2145       if ($validClauses > 1) {
2146         $this->_useDistinct = TRUE;
2147       }
2148     }
2149
2150     return implode(' AND ', $andClauses);
2151   }
2152
2153   /**
2154    * Generate where clause for any parameters not already handled.
2155    *
2156    * @param array $values
2157    *
2158    * @throws Exception
2159    */
2160   public function restWhere(&$values) {
2161     $name = CRM_Utils_Array::value(0, $values);
2162     $op = CRM_Utils_Array::value(1, $values);
2163     $value = CRM_Utils_Array::value(2, $values);
2164     $grouping = CRM_Utils_Array::value(3, $values);
2165     $wildcard = CRM_Utils_Array::value(4, $values);
2166
2167     if (isset($grouping) && empty($this->_where[$grouping])) {
2168       $this->_where[$grouping] = [];
2169     }
2170
2171     $multipleFields = ['url'];
2172
2173     //check if the location type exists for fields
2174     $lType = '';
2175     $locType = explode('-', $name);
2176
2177     if (!in_array($locType[0], $multipleFields)) {
2178       //add phone type if exists
2179       if (isset($locType[2]) && $locType[2]) {
2180         $locType[2] = CRM_Core_DAO::escapeString($locType[2]);
2181       }
2182     }
2183
2184     $field = CRM_Utils_Array::value($name, $this->_fields);
2185
2186     if (!$field) {
2187       $field = CRM_Utils_Array::value($locType[0], $this->_fields);
2188
2189       if (!$field) {
2190         // Strip any trailing _high & _low that might be appended.
2191         $realFieldName = str_replace(['_high', '_low'], '', $name);
2192         if (isset($this->_fields[$realFieldName])) {
2193           $field = $this->_fields[str_replace(['_high', '_low'], '', $realFieldName)];
2194           $this->dateQueryBuilder($values, $field['table_name'], $realFieldName, $realFieldName, $field['title']);
2195         }
2196         return;
2197       }
2198     }
2199
2200     $setTables = TRUE;
2201
2202     $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2203     if (isset($locType[1]) && is_numeric($locType[1])) {
2204       $lType = $locationType[$locType[1]];
2205     }
2206     if ($lType) {
2207       $field['title'] .= " ($lType)";
2208     }
2209
2210     if (substr($name, 0, 14) === 'state_province') {
2211       if (isset($locType[1]) && is_numeric($locType[1])) {
2212         $setTables = FALSE;
2213         $aName = "{$lType}-address";
2214         $where = "`$aName`.state_province_id";
2215       }
2216       else {
2217         $where = "civicrm_address.state_province_id";
2218       }
2219
2220       $this->_where[$grouping][] = self::buildClause($where, $op, $value);
2221       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2222       list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
2223       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2224     }
2225     elseif (!empty($field['pseudoconstant'])) {
2226       $this->optionValueQuery(
2227         $name, $op, $value, $grouping,
2228         'CRM_Contact_DAO_Contact',
2229         $field,
2230         $field['title'],
2231         'String',
2232         TRUE
2233       );
2234       if ($name == 'gender_id') {
2235         self::$_openedPanes[ts('Demographics')] = TRUE;
2236       }
2237     }
2238     elseif (substr($name, 0, 7) === 'country' || substr($name, 0, 6) === 'county') {
2239       $name = (substr($name, 0, 7) === 'country') ? "country_id" : "county_id";
2240       if (isset($locType[1]) && is_numeric($locType[1])) {
2241         $setTables = FALSE;
2242         $aName = "{$lType}-address";
2243         $where = "`$aName`.$name";
2244       }
2245       else {
2246         $where = "civicrm_address.$name";
2247       }
2248
2249       $this->_where[$grouping][] = self::buildClause($where, $op, $value, 'Positive');
2250       $this->_tables[$aName] = $this->_whereTables[$aName] = 1;
2251
2252       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
2253       $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $field['title'], 2 => $qillop, 3 => $qillVal]);
2254     }
2255     elseif ($name === 'world_region') {
2256       $this->optionValueQuery(
2257         $name, $op, $value, $grouping,
2258         NULL,
2259         $field,
2260         ts('World Region'),
2261         'Positive',
2262         TRUE
2263       );
2264     }
2265     elseif ($name === 'is_deceased') {
2266       $this->setQillAndWhere($name, $op, $value, $grouping, $field);
2267       self::$_openedPanes[ts('Demographics')] = TRUE;
2268     }
2269     elseif ($name === 'created_date' || $name === 'modified_date' || $name === 'deceased_date' || $name === 'birth_date') {
2270       $appendDateTime = TRUE;
2271       if ($name === 'deceased_date' || $name === 'birth_date') {
2272         $appendDateTime = FALSE;
2273         self::$_openedPanes[ts('Demographics')] = TRUE;
2274       }
2275       $this->dateQueryBuilder($values, 'contact_a', $name, $name, $field['title'], $appendDateTime);
2276     }
2277     elseif ($name === 'contact_id') {
2278       if (is_int($value)) {
2279         $this->_where[$grouping][] = self::buildClause($field['where'], $op, $value);
2280         $this->_qill[$grouping][] = "$field[title] $op $value";
2281       }
2282     }
2283     elseif ($name === 'name') {
2284       $value = CRM_Core_DAO::escapeString($value);
2285       if ($wildcard) {
2286         $op = 'LIKE';
2287         $value = self::getWildCardedValue($wildcard, $op, $value);
2288       }
2289       CRM_Core_Error::deprecatedFunctionWarning('Untested code path');
2290       // @todo it's likely this code path is obsolete / never called. It is definitely not
2291       // passed through in our test suite.
2292       $this->_where[$grouping][] = self::buildClause($field['where'], $op, "'$value'");
2293       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2294     }
2295     elseif ($name === 'current_employer') {
2296       if ($wildcard) {
2297         $op = 'LIKE';
2298         $value = self::getWildCardedValue($wildcard, $op, $value);
2299       }
2300       $ceWhereClause = self::buildClause("contact_a.organization_name", $op,
2301         $value
2302       );
2303       $ceWhereClause .= " AND contact_a.contact_type = 'Individual'";
2304       $this->_where[$grouping][] = $ceWhereClause;
2305       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2306     }
2307     elseif ($name === 'email_greeting') {
2308       CRM_Core_Error::deprecatedFunctionWarning('pass in email_greeting_id or email_greeting_display');
2309       $filterCondition = ['greeting_type' => 'email_greeting'];
2310       $this->optionValueQuery(
2311         $name, $op, $value, $grouping,
2312         CRM_Core_PseudoConstant::greeting($filterCondition),
2313         $field,
2314         ts('Email Greeting')
2315       );
2316     }
2317     elseif ($name === 'postal_greeting') {
2318       CRM_Core_Error::deprecatedFunctionWarning('pass in postal_greeting_id or postal_greeting_display');
2319       $filterCondition = ['greeting_type' => 'postal_greeting'];
2320       $this->optionValueQuery(
2321         $name, $op, $value, $grouping,
2322         CRM_Core_PseudoConstant::greeting($filterCondition),
2323         $field,
2324         ts('Postal Greeting')
2325       );
2326     }
2327     elseif ($name === 'addressee') {
2328       CRM_Core_Error::deprecatedFunctionWarning('pass in addressee_id or addressee_display');
2329       $filterCondition = ['greeting_type' => 'addressee'];
2330       $this->optionValueQuery(
2331         $name, $op, $value, $grouping,
2332         CRM_Core_PseudoConstant::greeting($filterCondition),
2333         $field,
2334         ts('Addressee')
2335       );
2336     }
2337     elseif (substr($name, 0, 4) === 'url-') {
2338       $tName = 'civicrm_website';
2339       $this->_whereTables[$tName] = $this->_tables[$tName] = "\nLEFT JOIN civicrm_website ON ( civicrm_website.contact_id = contact_a.id )";
2340       $value = CRM_Core_DAO::escapeString($value);
2341       if ($wildcard) {
2342         $op = 'LIKE';
2343         $value = self::getWildCardedValue($wildcard, $op, $value);
2344       }
2345
2346       $this->_where[$grouping][] = $d = self::buildClause('civicrm_website.url', $op, $value);
2347       $this->_qill[$grouping][] = "$field[title] $op \"$value\"";
2348     }
2349     elseif ($name === 'contact_is_deleted') {
2350       $this->setQillAndWhere('is_deleted', $op, $value, $grouping, $field);
2351     }
2352     elseif (!empty($field['where'])) {
2353       $type = NULL;
2354       if (!empty($field['type'])) {
2355         $type = CRM_Utils_Type::typeToString($field['type']);
2356       }
2357
2358       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2359
2360       if (isset($locType[1]) &&
2361         is_numeric($locType[1])
2362       ) {
2363         $setTables = FALSE;
2364
2365         //get the location name
2366         list($tName, $fldName) = self::getLocationTableName($field['where'], $locType);
2367         $fieldName = "`$tName`.$fldName";
2368
2369         // we set both _tables & whereTables because whereTables doesn't seem to do what the name implies it should
2370         $this->_tables[$tName] = $this->_whereTables[$tName] = 1;
2371
2372       }
2373       else {
2374         if ($tableName == 'civicrm_contact') {
2375           $fieldName = "contact_a.{$fieldName}";
2376         }
2377         else {
2378           $fieldName = $field['where'];
2379         }
2380       }
2381
2382       list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, $field['title'], $value, $op);
2383       $this->_qill[$grouping][] = ts("%1 %2 %3", [
2384         1 => $field['title'],
2385         2 => $qillop,
2386         3 => (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) ? $qillVal : "'$qillVal'",
2387       ]);
2388
2389       if (is_array($value)) {
2390         // traditionally an array being passed has been a fatal error. We can take advantage of this to add support
2391         // for api style operators for functions that hit this point without worrying about regression
2392         // (the previous comments indicated the condition for hitting this point were unknown
2393         // per CRM-14743 we are adding modified_date & created_date operator support
2394         $operations = array_keys($value);
2395         foreach ($operations as $operator) {
2396           if (!in_array($operator, CRM_Core_DAO::acceptedSQLOperators())) {
2397             //Via Contact get api value is not in array(operator => array(values)) format ONLY for IN/NOT IN operators
2398             //so this condition will satisfy the search for now
2399             if (strpos($op, 'IN') !== FALSE) {
2400               $value = [$op => $value];
2401             }
2402             // we don't know when this might happen
2403             else {
2404               CRM_Core_Error::fatal(ts("%1 is not a valid operator", [1 => $operator]));
2405             }
2406           }
2407         }
2408         $this->_where[$grouping][] = CRM_Core_DAO::createSQLFilter($fieldName, $value, $type);
2409       }
2410       else {
2411         if ($wildcard) {
2412           $op = 'LIKE';
2413           $value = self::getWildCardedValue($wildcard, $op, $value);
2414         }
2415
2416         $this->_where[$grouping][] = self::buildClause($fieldName, $op, $value, $type);
2417       }
2418     }
2419
2420     if ($setTables && isset($field['where'])) {
2421       list($tableName, $fieldName) = explode('.', $field['where'], 2);
2422       if (isset($tableName)) {
2423         $this->_tables[$tableName] = 1;
2424         $this->_whereTables[$tableName] = 1;
2425       }
2426     }
2427   }
2428
2429   /**
2430    * @param $where
2431    * @param $locType
2432    *
2433    * @return array
2434    * @throws Exception
2435    */
2436   public static function getLocationTableName(&$where, &$locType) {
2437     if (isset($locType[1]) && is_numeric($locType[1])) {
2438       list($tbName, $fldName) = explode(".", $where);
2439
2440       //get the location name
2441       $locationType = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2442       $specialFields = ['email', 'im', 'phone', 'openid', 'phone_ext'];
2443       if (in_array($locType[0], $specialFields)) {
2444         //hack to fix / special handing for phone_ext
2445         if ($locType[0] == 'phone_ext') {
2446           $locType[0] = 'phone';
2447         }
2448         if (isset($locType[2]) && $locType[2]) {
2449           $tName = "{$locationType[$locType[1]]}-{$locType[0]}-{$locType[2]}";
2450         }
2451         else {
2452           $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2453         }
2454       }
2455       elseif (in_array($locType[0],
2456         [
2457           'address_name',
2458           'street_address',
2459           'street_name',
2460           'street_number_suffix',
2461           'street_unit',
2462           'supplemental_address_1',
2463           'supplemental_address_2',
2464           'supplemental_address_3',
2465           'city',
2466           'postal_code',
2467           'postal_code_suffix',
2468           'geo_code_1',
2469           'geo_code_2',
2470           'master_id',
2471         ]
2472       )) {
2473         //fix for search by profile with address fields.
2474         $tName = "{$locationType[$locType[1]]}-address";
2475       }
2476       elseif (in_array($locType[0],
2477           [
2478             'on_hold',
2479             'signature_html',
2480             'signature_text',
2481             'is_bulkmail',
2482           ]
2483         )) {
2484         $tName = "{$locationType[$locType[1]]}-email";
2485       }
2486       elseif ($locType[0] == 'provider_id') {
2487         $tName = "{$locationType[$locType[1]]}-im";
2488       }
2489       elseif ($locType[0] == 'openid') {
2490         $tName = "{$locationType[$locType[1]]}-openid";
2491       }
2492       else {
2493         $tName = "{$locationType[$locType[1]]}-{$locType[0]}";
2494       }
2495       $tName = str_replace(' ', '_', $tName);
2496       return [$tName, $fldName];
2497     }
2498     CRM_Core_Error::fatal();
2499   }
2500
2501   /**
2502    * Given a result dao, extract the values and return that array
2503    *
2504    * @param CRM_Core_DAO $dao
2505    *
2506    * @return array
2507    *   values for this query
2508    */
2509   public function store($dao) {
2510     $value = [];
2511
2512     foreach ($this->_element as $key => $dontCare) {
2513       if (property_exists($dao, $key)) {
2514         if (strpos($key, '-') !== FALSE) {
2515           $values = explode('-', $key);
2516           $lastElement = array_pop($values);
2517           $current = &$value;
2518           $cnt = count($values);
2519           $count = 1;
2520           foreach ($values as $v) {
2521             if (!array_key_exists($v, $current)) {
2522               $current[$v] = [];
2523             }
2524             //bad hack for im_provider
2525             if ($lastElement == 'provider_id') {
2526               if ($count < $cnt) {
2527                 $current = &$current[$v];
2528               }
2529               else {
2530                 $lastElement = "{$v}_{$lastElement}";
2531               }
2532             }
2533             else {
2534               $current = &$current[$v];
2535             }
2536             $count++;
2537           }
2538
2539           $current[$lastElement] = $dao->$key;
2540         }
2541         else {
2542           $value[$key] = $dao->$key;
2543         }
2544       }
2545     }
2546     return $value;
2547   }
2548
2549   /**
2550    * Getter for tables array.
2551    *
2552    * @return array
2553    */
2554   public function tables() {
2555     return $this->_tables;
2556   }
2557
2558   /**
2559    * Sometimes used to create the from clause, but, not reliably, set
2560    * this AND set tables.
2561    *
2562    * It's unclear the intent - there is a 'simpleFrom' clause which
2563    * takes whereTables into account & a fromClause which doesn't.
2564    *
2565    * logic may have eroded?
2566    *
2567    * @return array
2568    */
2569   public function whereTables() {
2570     return $this->_whereTables;
2571   }
2572
2573   /**
2574    * Generate the where clause (used in match contacts and permissions)
2575    *
2576    * @param array $params
2577    * @param array $fields
2578    * @param array $tables
2579    * @param $whereTables
2580    * @param bool $strict
2581    *
2582    * @return string
2583    */
2584   public static function getWhereClause($params, $fields, &$tables, &$whereTables, $strict = FALSE) {
2585     $query = new CRM_Contact_BAO_Query($params, NULL, $fields,
2586       FALSE, $strict
2587     );
2588
2589     $tables = array_merge($query->tables(), $tables);
2590     $whereTables = array_merge($query->whereTables(), $whereTables);
2591
2592     return $query->_whereClause;
2593   }
2594
2595   /**
2596    * Create the from clause.
2597    *
2598    * @param array $tables
2599    *   Tables that need to be included in this from clause. If null,
2600    *   return mimimal from clause (i.e. civicrm_contact).
2601    * @param array $inner
2602    *   Tables that should be inner-joined.
2603    * @param array $right
2604    *   Tables that should be right-joined.
2605    * @param bool $primaryLocation
2606    *   Search on primary location. See note below.
2607    * @param int $mode
2608    *   Determines search mode based on bitwise MODE_* constants.
2609    * @param string|NULL $apiEntity
2610    *   Determines search mode based on entity by string.
2611    *
2612    * The $primaryLocation flag only seems to be used when
2613    * locationType() has been called. This may be a search option
2614    * exposed, or perhaps it's a "search all details" approach which
2615    * predates decoupling of location types and primary fields?
2616    *
2617    * @see https://issues.civicrm.org/jira/browse/CRM-19967
2618    *
2619    * @return string
2620    *   the from clause
2621    */
2622   public static function fromClause(&$tables, $inner = NULL, $right = NULL, $primaryLocation = TRUE, $mode = 1, $apiEntity = NULL) {
2623
2624     $from = ' FROM civicrm_contact contact_a';
2625     if (empty($tables)) {
2626       return $from;
2627     }
2628
2629     if (!empty($tables['civicrm_worldregion'])) {
2630       $tables = array_merge(['civicrm_country' => 1], $tables);
2631     }
2632
2633     if ((!empty($tables['civicrm_state_province']) || !empty($tables['civicrm_country']) ||
2634         CRM_Utils_Array::value('civicrm_county', $tables)) && empty($tables['civicrm_address'])) {
2635       $tables = array_merge(['civicrm_address' => 1],
2636         $tables
2637       );
2638     }
2639
2640     // add group_contact and group table is subscription history is present
2641     if (!empty($tables['civicrm_subscription_history']) && empty($tables['civicrm_group'])) {
2642       $tables = array_merge([
2643         'civicrm_group' => 1,
2644         'civicrm_group_contact' => 1,
2645       ],
2646         $tables
2647       );
2648     }
2649
2650     // to handle table dependencies of components
2651     CRM_Core_Component::tableNames($tables);
2652     // to handle table dependencies of hook injected tables
2653     CRM_Contact_BAO_Query_Hook::singleton()->setTableDependency($tables);
2654
2655     //format the table list according to the weight
2656     $info = CRM_Core_TableHierarchy::info();
2657
2658     foreach ($tables as $key => $value) {
2659       $k = 99;
2660       if (strpos($key, '-') !== FALSE) {
2661         $keyArray = explode('-', $key);
2662         $k = CRM_Utils_Array::value('civicrm_' . $keyArray[1], $info, 99);
2663       }
2664       elseif (strpos($key, '_') !== FALSE) {
2665         $keyArray = explode('_', $key);
2666         if (is_numeric(array_pop($keyArray))) {
2667           $k = CRM_Utils_Array::value(implode('_', $keyArray), $info, 99);
2668         }
2669         else {
2670           $k = CRM_Utils_Array::value($key, $info, 99);
2671         }
2672       }
2673       else {
2674         $k = CRM_Utils_Array::value($key, $info, 99);
2675       }
2676       $tempTable[$k . ".$key"] = $key;
2677     }
2678     ksort($tempTable);
2679     $newTables = [];
2680     foreach ($tempTable as $key) {
2681       $newTables[$key] = $tables[$key];
2682     }
2683
2684     $tables = $newTables;
2685
2686     foreach ($tables as $name => $value) {
2687       if (!$value) {
2688         continue;
2689       }
2690
2691       if (!empty($inner[$name])) {
2692         $side = 'INNER';
2693       }
2694       elseif (!empty($right[$name])) {
2695         $side = 'RIGHT';
2696       }
2697       else {
2698         $side = 'LEFT';
2699       }
2700
2701       if ($value != 1) {
2702         // if there is already a join statement in value, use value itself
2703         if (strpos($value, 'JOIN')) {
2704           $from .= " $value ";
2705         }
2706         else {
2707           $from .= " $side JOIN $name ON ( $value ) ";
2708         }
2709         continue;
2710       }
2711
2712       $from .= ' ' . trim(self::getEntitySpecificJoins($name, $mode, $side, $primaryLocation)) . ' ';
2713     }
2714     return $from;
2715   }
2716
2717   /**
2718    * Get join statements for the from clause depending on entity type
2719    *
2720    * @param string $name
2721    * @param int $mode
2722    * @param string $side
2723    * @param string $primaryLocation
2724    * @return string
2725    */
2726   protected static function getEntitySpecificJoins($name, $mode, $side, $primaryLocation) {
2727     $limitToPrimaryClause = $primaryLocation ? "AND {$name}.is_primary = 1" : '';
2728     switch ($name) {
2729       case 'civicrm_address':
2730         //CRM-14263 further handling of address joins further down...
2731         return " $side JOIN civicrm_address ON ( contact_a.id = civicrm_address.contact_id {$limitToPrimaryClause} )";
2732
2733       case 'civicrm_state_province':
2734         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2735         // but that will have been implied-in by the calling function.
2736         // test cover in testContactIDQuery
2737         return " $side JOIN civicrm_state_province ON ( civicrm_address.state_province_id = civicrm_state_province.id )";
2738
2739       case 'civicrm_country':
2740         // This is encountered when doing an export after having applied a 'sort' - it pretty much implies primary
2741         // but that will have been implied-in by the calling function.
2742         // test cover in testContactIDQuery
2743         return " $side JOIN civicrm_country ON ( civicrm_address.country_id = civicrm_country.id )";
2744
2745       case 'civicrm_phone':
2746         return " $side JOIN civicrm_phone ON (contact_a.id = civicrm_phone.contact_id {$limitToPrimaryClause}) ";
2747
2748       case 'civicrm_email':
2749         return " $side JOIN civicrm_email ON (contact_a.id = civicrm_email.contact_id {$limitToPrimaryClause})";
2750
2751       case 'civicrm_im':
2752         return " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id {$limitToPrimaryClause}) ";
2753
2754       case 'im_provider':
2755         $from = " $side JOIN civicrm_im ON (contact_a.id = civicrm_im.contact_id) ";
2756         $from .= " $side JOIN civicrm_option_group option_group_imProvider ON option_group_imProvider.name = 'instant_messenger_service'";
2757         $from .= " $side JOIN civicrm_option_value im_provider ON (civicrm_im.provider_id = im_provider.value AND option_group_imProvider.id = im_provider.option_group_id)";
2758         return $from;
2759
2760       case 'civicrm_openid':
2761         return " $side JOIN civicrm_openid ON ( civicrm_openid.contact_id = contact_a.id {$limitToPrimaryClause} )";
2762
2763       case 'civicrm_worldregion':
2764         // We can be sure from the calling function that country will already be joined in.
2765         // we really don't need world_region - we could use a pseudoconstant for it.
2766         return " $side JOIN civicrm_worldregion ON civicrm_country.region_id = civicrm_worldregion.id ";
2767
2768       case 'civicrm_location_type':
2769         return " $side JOIN civicrm_location_type ON civicrm_address.location_type_id = civicrm_location_type.id ";
2770
2771       case 'civicrm_group':
2772         return " $side JOIN civicrm_group ON civicrm_group.id = civicrm_group_contact.group_id ";
2773
2774       case 'civicrm_group_contact':
2775         return " $side JOIN civicrm_group_contact ON contact_a.id = civicrm_group_contact.contact_id ";
2776
2777       case 'civicrm_group_contact_cache':
2778         return " $side JOIN civicrm_group_contact_cache ON contact_a.id = civicrm_group_contact_cache.contact_id ";
2779
2780       case 'civicrm_activity':
2781       case 'civicrm_activity_tag':
2782       case 'activity_type':
2783       case 'activity_status':
2784       case 'parent_id':
2785       case 'civicrm_activity_contact':
2786       case 'source_contact':
2787       case 'activity_priority':
2788         return CRM_Activity_BAO_Query::from($name, $mode, $side);
2789
2790       case 'civicrm_entity_tag':
2791         $from = " $side JOIN civicrm_entity_tag ON ( civicrm_entity_tag.entity_table = 'civicrm_contact'";
2792         return "$from AND civicrm_entity_tag.entity_id = contact_a.id ) ";
2793
2794       case 'civicrm_note':
2795         $from = " $side JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact'";
2796         return "$from AND contact_a.id = civicrm_note.entity_id ) ";
2797
2798       case 'civicrm_subscription_history':
2799         $from = " $side JOIN civicrm_subscription_history";
2800         $from .= " ON civicrm_group_contact.contact_id = civicrm_subscription_history.contact_id";
2801         return "$from AND civicrm_group_contact.group_id =  civicrm_subscription_history.group_id";
2802
2803       case 'civicrm_relationship':
2804         if (self::$_relType == 'reciprocal') {
2805           if (self::$_relationshipTempTable) {
2806             // we have a temptable to join on
2807             $tbl = self::$_relationshipTempTable;
2808             return " INNER JOIN {$tbl} civicrm_relationship ON civicrm_relationship.contact_id = contact_a.id";
2809           }
2810           else {
2811             $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id OR civicrm_relationship.contact_id_a = contact_a.id)";
2812             $from .= " $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id OR civicrm_relationship.contact_id_b = contact_b.id)";
2813             return $from;
2814           }
2815         }
2816         elseif (self::$_relType == 'b') {
2817           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_b = contact_a.id )";
2818           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_a = contact_b.id )";
2819         }
2820         else {
2821           $from = " $side JOIN civicrm_relationship ON (civicrm_relationship.contact_id_a = contact_a.id )";
2822           return "$from $side JOIN civicrm_contact contact_b ON (civicrm_relationship.contact_id_b = contact_b.id )";
2823         }
2824
2825       case 'civicrm_log':
2826         $from = " INNER JOIN civicrm_log ON (civicrm_log.entity_id = contact_a.id AND civicrm_log.entity_table = 'civicrm_contact')";
2827         return "$from INNER JOIN civicrm_contact contact_b_log ON (civicrm_log.modified_id = contact_b_log.id)";
2828
2829       case 'civicrm_tag':
2830         return " $side  JOIN civicrm_tag ON civicrm_entity_tag.tag_id = civicrm_tag.id ";
2831
2832       case 'civicrm_grant':
2833         return CRM_Grant_BAO_Query::from($name, $mode, $side);
2834
2835       case 'civicrm_website':
2836         return " $side JOIN civicrm_website ON contact_a.id = civicrm_website.contact_id ";
2837
2838       case 'civicrm_campaign':
2839         //Move to default case if not in either mode.
2840         if ($mode & CRM_Contact_BAO_Query::MODE_CONTRIBUTE) {
2841           return CRM_Contribute_BAO_Query::from($name, $mode, $side);
2842         }
2843         elseif ($mode & CRM_Contact_BAO_Query::MODE_MAILING) {
2844           return CRM_Mailing_BAO_Query::from($name, $mode, $side);
2845         }
2846         elseif ($mode & CRM_Contact_BAO_Query::MODE_CAMPAIGN) {
2847           return CRM_Campaign_BAO_Query::from($name, $mode, $side);
2848         }
2849
2850       default:
2851         $locationTypeName = '';
2852         if (strpos($name, '-address') != 0) {
2853           $locationTypeName = 'address';
2854         }
2855         elseif (strpos($name, '-phone') != 0) {
2856           $locationTypeName = 'phone';
2857         }
2858         elseif (strpos($name, '-email') != 0) {
2859           $locationTypeName = 'email';
2860         }
2861         elseif (strpos($name, '-im') != 0) {
2862           $locationTypeName = 'im';
2863         }
2864         elseif (strpos($name, '-openid') != 0) {
2865           $locationTypeName = 'openid';
2866         }
2867
2868         if ($locationTypeName) {
2869           //we have a join on an location table - possibly in conjunction with search builder - CRM-14263
2870           $parts = explode('-', $name);
2871           $locationTypes = CRM_Core_DAO_Address::buildOptions('location_type_id', 'validate');
2872           foreach ($locationTypes as $locationTypeID => $locationType) {
2873             if ($parts[0] == str_replace(' ', '_', $locationType)) {
2874               $locationID = $locationTypeID;
2875             }
2876           }
2877           $from = " $side JOIN civicrm_{$locationTypeName} `{$name}` ON ( contact_a.id = `{$name}`.contact_id ) and `{$name}`.location_type_id = $locationID ";
2878         }
2879         else {
2880           $from = CRM_Core_Component::from($name, $mode, $side);
2881         }
2882         $from .= CRM_Contact_BAO_Query_Hook::singleton()->buildSearchfrom($name, $mode, $side);
2883
2884         return $from;
2885     }
2886   }
2887
2888   /**
2889    * WHERE / QILL clause for deleted_contacts
2890    *
2891    * @param array $values
2892    */
2893   public function deletedContacts($values) {
2894     list($_, $_, $value, $grouping, $_) = $values;
2895     if ($value) {
2896       // *prepend* to the relevant grouping as this is quite an important factor
2897       array_unshift($this->_qill[$grouping], ts('Search in Trash'));
2898     }
2899   }
2900
2901   /**
2902    * Where / qill clause for contact_type
2903    *
2904    * @param $values
2905    */
2906   public function contactType(&$values) {
2907     list($name, $op, $value, $grouping, $wildcard) = $values;
2908
2909     $subTypes = [];
2910     $clause = [];
2911
2912     // account for search builder mapping multiple values
2913     if (!is_array($value)) {
2914       $values = self::parseSearchBuilderString($value, 'String');
2915       if (is_array($values)) {
2916         $value = array_flip($values);
2917       }
2918     }
2919
2920     if (is_array($value)) {
2921       foreach ($value as $k => $v) {
2922         // fix for CRM-771
2923         if ($k) {
2924           $subType = NULL;
2925           $contactType = $k;
2926           if (strpos($k, CRM_Core_DAO::VALUE_SEPARATOR)) {
2927             list($contactType, $subType) = explode(CRM_Core_DAO::VALUE_SEPARATOR, $k, 2);
2928           }
2929
2930           if (!empty($subType)) {
2931             $subTypes[$subType] = 1;
2932           }
2933           $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2934         }
2935       }
2936     }
2937     else {
2938       $contactTypeANDSubType = explode(CRM_Core_DAO::VALUE_SEPARATOR, $value, 2);
2939       $contactType = $contactTypeANDSubType[0];
2940       $subType = CRM_Utils_Array::value(1, $contactTypeANDSubType);
2941       if (!empty($subType)) {
2942         $subTypes[$subType] = 1;
2943       }
2944       $clause[$contactType] = "'" . CRM_Utils_Type::escape($contactType, 'String') . "'";
2945     }
2946
2947     // fix for CRM-771
2948     if (!empty($clause)) {
2949       $quill = $clause;
2950       if ($op == 'IN' || $op == 'NOT IN') {
2951         $this->_where[$grouping][] = "contact_a.contact_type $op (" . implode(',', $clause) . ')';
2952       }
2953       else {
2954         $type = array_pop($clause);
2955         $this->_where[$grouping][] = self::buildClause("contact_a.contact_type", $op, $contactType);
2956       }
2957
2958       $this->_qill[$grouping][] = ts('Contact Type') . " $op " . implode(' ' . ts('or') . ' ', $quill);
2959
2960       if (!empty($subTypes)) {
2961         $this->includeContactSubTypes($subTypes, $grouping);
2962       }
2963     }
2964   }
2965
2966   /**
2967    * Where / qill clause for contact_sub_type.
2968    *
2969    * @param array $values
2970    */
2971   public function contactSubType(&$values) {
2972     list($name, $op, $value, $grouping, $wildcard) = $values;
2973     $this->includeContactSubTypes($value, $grouping, $op);
2974   }
2975
2976   /**
2977    * @param $value
2978    * @param $grouping
2979    * @param string $op
2980    */
2981   public function includeContactSubTypes($value, $grouping, $op = 'LIKE') {
2982
2983     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
2984       $op = key($value);
2985       $value = $value[$op];
2986     }
2987
2988     $clause = [];
2989     $alias = "contact_a.contact_sub_type";
2990     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
2991
2992     $op = str_replace('IN', 'LIKE', $op);
2993     $op = str_replace('=', 'LIKE', $op);
2994     $op = str_replace('!', 'NOT ', $op);
2995
2996     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
2997       $this->_where[$grouping][] = self::buildClause($alias, $op, $value, 'String');
2998     }
2999     elseif (is_array($value)) {
3000       foreach ($value as $k => $v) {
3001         $clause[$k] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($v, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
3002       }
3003     }
3004     else {
3005       $clause[$value] = "($alias $op '%" . CRM_Core_DAO::VALUE_SEPARATOR . CRM_Utils_Type::escape($value, 'String') . CRM_Core_DAO::VALUE_SEPARATOR . "%')";
3006     }
3007
3008     if (!empty($clause)) {
3009       $this->_where[$grouping][] = "( " . implode(' OR ', $clause) . " )";
3010     }
3011     $this->_qill[$grouping][] = ts('Contact Subtype %1 ', [1 => $qillOperators[$op]]) . implode(' ' . ts('or') . ' ', array_keys($clause));
3012   }
3013
3014   /**
3015    * Where / qill clause for groups.
3016    *
3017    * @param $values
3018    *
3019    * @throws \CRM_Core_Exception
3020    */
3021   public function group($values) {
3022     list($name, $op, $value, $grouping, $wildcard) = $values;
3023
3024     // If the $value is in OK (operator as key) array format we need to extract the key as operator and value first
3025     if (is_array($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3026       $op = key($value);
3027       $value = $value[$op];
3028     }
3029     // Translate EMPTY to NULL as EMPTY is cannot be used in it's intended meaning here
3030     // so has to be 'squashed into' NULL. (ie. group membership cannot be '').
3031     // even one group might equate to multiple when looking at children so IN is simpler.
3032     // @todo - also look at != casting but there are rows below to review.
3033     $opReplacements = [
3034       'EMPTY' => 'NULL',
3035       'NOT EMPTY' => 'NOT NULL',
3036       '=' => 'IN',
3037     ];
3038     if (isset($opReplacements[$op])) {
3039       $op = $opReplacements[$op];
3040     }
3041
3042     if (strpos($op, 'NULL')) {
3043       $value = NULL;
3044     }
3045
3046     if (is_array($value) && count($value) > 1) {
3047       if (strpos($op, 'IN') === FALSE && strpos($op, 'NULL') === FALSE) {
3048         CRM_Core_Error::fatal(ts("%1 is not a valid operator", [1 => $op]));
3049       }
3050       $this->_useDistinct = TRUE;
3051     }
3052
3053     if (isset($value)) {
3054       $value = CRM_Utils_Array::value($op, $value, $value);
3055     }
3056
3057     if ($name == 'group_type') {
3058       $value = array_keys($this->getGroupsFromTypeCriteria($value));
3059     }
3060
3061     $regularGroupIDs = $smartGroupIDs = [];
3062     foreach ((array) $value as $id) {
3063       if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3064         $smartGroupIDs[] = (int) $id;
3065       }
3066       else {
3067         $regularGroupIDs[] = (int) trim($id);
3068       }
3069     }
3070     $hasNonSmartGroups = count($regularGroupIDs);
3071
3072     $isNotOp = ($op == 'NOT IN' || $op == '!=');
3073
3074     $statusJoinClause = $this->getGroupStatusClause($grouping);
3075     // If we are searching for 'Removed' contacts then despite it being a smart group we only care about the group_contact table.
3076     $isGroupStatusSearch = (!empty($this->getSelectedGroupStatuses($grouping)) && $this->getSelectedGroupStatuses($grouping) !== ["'Added'"]);
3077     $groupClause = [];
3078     if ($hasNonSmartGroups || empty($value) || $isGroupStatusSearch) {
3079       // include child groups IDs if any
3080       $childGroupIds = (array) CRM_Contact_BAO_Group::getChildGroupIds($regularGroupIDs);
3081       foreach ($childGroupIds as $key => $id) {
3082         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Group', $id, 'saved_search_id')) {
3083           $smartGroupIDs[] = $id;
3084           unset($childGroupIds[$key]);
3085         }
3086       }
3087       if (count($childGroupIds)) {
3088         $regularGroupIDs = array_merge($regularGroupIDs, $childGroupIds);
3089       }
3090
3091       if (empty($regularGroupIDs)) {
3092         if ($isGroupStatusSearch) {
3093           $regularGroupIDs = $smartGroupIDs;
3094         }
3095         // If it is still empty we want a filter that blocks all results.
3096         if (empty($regularGroupIDs)) {
3097           $regularGroupIDs = [0];
3098         }
3099       }
3100
3101       $gcTable = "`civicrm_group_contact-" . uniqid() . "`";
3102       $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3103
3104       // @todo consider just casting != to NOT IN & handling both together.
3105       if ($op == '!=') {
3106         $groupIds = '';
3107         if (!empty($regularGroupIDs)) {
3108           $groupIds = CRM_Utils_Type::validate(
3109             implode(',', (array) $regularGroupIDs),
3110             'CommaSeparatedIntegers'
3111           );
3112         }
3113         $clause = "{$gcTable}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact cgc WHERE cgc.group_id = $groupIds )";
3114       }
3115       else {
3116         $clause = self::buildClause("{$gcTable}.group_id", $op, $regularGroupIDs);
3117       }
3118       $groupClause[] = "( {$clause} )";
3119
3120       if ($statusJoinClause) {
3121         $joinClause[] = "{$gcTable}.$statusJoinClause";
3122       }
3123       $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3124     }
3125
3126     //CRM-19589: contact(s) removed from a Smart Group, resides in civicrm_group_contact table
3127     // If we are only searching for Removed or Pending contacts we don't need to resolve the smart group
3128     // as that info is in the group_contact table.
3129     if ((count($smartGroupIDs) || empty($value)) && !$isGroupStatusSearch) {
3130       $this->_groupUniqueKey = uniqid();
3131       $this->_groupKeys[] = $this->_groupUniqueKey;
3132       $gccTableAlias = "civicrm_group_contact_cache_{$this->_groupUniqueKey}";
3133       $groupContactCacheClause = $this->addGroupContactCache($smartGroupIDs, $gccTableAlias, "contact_a", $op);
3134       if (!empty($groupContactCacheClause)) {
3135         if ($isNotOp) {
3136           $groupIds = implode(',', (array) $smartGroupIDs);
3137           $gcTable = "civicrm_group_contact_{$this->_groupUniqueKey}";
3138           $joinClause = ["contact_a.id = {$gcTable}.contact_id"];
3139           $this->_tables[$gcTable] = $this->_whereTables[$gcTable] = " LEFT JOIN civicrm_group_contact {$gcTable} ON (" . implode(' AND ', $joinClause) . ")";
3140           if (strpos($op, 'IN') !== FALSE) {
3141             $groupClause[] = "{$gcTable}.group_id $op ( $groupIds ) AND {$gccTableAlias}.group_id IS NULL";
3142           }
3143           else {
3144             $groupClause[] = "{$gcTable}.group_id $op $groupIds AND {$gccTableAlias}.group_id IS NULL";
3145           }
3146         }
3147         $groupClause[] = " ( {$groupContactCacheClause} ) ";
3148       }
3149     }
3150
3151     $and = ($op == 'IS NULL') ? ' AND ' : ' OR ';
3152     if (!empty($groupClause)) {
3153       $this->_where[$grouping][] = ' ( ' . implode($and, $groupClause) . ' ) ';
3154     }
3155
3156     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Group', 'id', $value, $op);
3157     $this->_qill[$grouping][] = ts("Group(s) %1 %2", [1 => $qillop, 2 => $qillVal]);
3158     if (strpos($op, 'NULL') === FALSE) {
3159       $this->_qill[$grouping][] = ts("Group Status %1", [1 => implode(' ' . ts('or') . ' ', $this->getSelectedGroupStatuses($grouping))]);
3160     }
3161   }
3162
3163   public function getGroupCacheTableKeys() {
3164     return $this->_groupKeys;
3165   }
3166
3167   /**
3168    * Function translates selection of group type into a list of groups.
3169    * @param $value
3170    *
3171    * @return array
3172    */
3173   public function getGroupsFromTypeCriteria($value) {
3174     $groupIds = [];
3175     foreach ((array) $value as $groupTypeValue) {
3176       $groupList = CRM_Core_PseudoConstant::group($groupTypeValue);
3177       $groupIds = ($groupIds + $groupList);
3178     }
3179     return $groupIds;
3180   }
3181
3182   /**
3183    * Prime smart group cache for smart groups in the search, and join
3184    * civicrm_group_contact_cache table into the query.
3185    *
3186    * @param array $groups IDs of groups specified in search criteria.
3187    * @param string $tableAlias Alias to use for civicrm_group_contact_cache table.
3188    * @param string $joinTable Table on which to join civicrm_group_contact_cache
3189    * @param string $op SQL comparison operator (NULL, IN, !=, IS NULL, etc.)
3190    * @param string $joinColumn Column in $joinTable on which to join civicrm_group_contact_cache.contact_id
3191    *
3192    * @return string WHERE clause component for smart group criteria.
3193    */
3194   public function addGroupContactCache($groups, $tableAlias, $joinTable = "contact_a", $op, $joinColumn = 'id') {
3195     $isNullOp = (strpos($op, 'NULL') !== FALSE);
3196     $groupsIds = $groups;
3197
3198     $operator = ['=' => 'IN', '!=' => 'NOT IN'];
3199     if (!empty($operator[$op]) && is_array($groups)) {
3200       $op = $operator[$op];
3201     }
3202     if (!$isNullOp && !$groups) {
3203       return NULL;
3204     }
3205     elseif (strpos($op, 'IN') !== FALSE) {
3206       $groups = [$op => $groups];
3207     }
3208     elseif (is_array($groups) && count($groups)) {
3209       $groups = ['IN' => $groups];
3210     }
3211
3212     // Find all the groups that are part of a saved search.
3213     $smartGroupClause = self::buildClause("id", $op, $groups, 'Int');
3214     $sql = "
3215 SELECT id, cache_date, saved_search_id, children
3216 FROM   civicrm_group
3217 WHERE  $smartGroupClause
3218   AND  ( saved_search_id != 0
3219    OR    saved_search_id IS NOT NULL
3220    OR    children IS NOT NULL )
3221 ";
3222
3223     $group = CRM_Core_DAO::executeQuery($sql);
3224
3225     while ($group->fetch()) {
3226       $this->_useDistinct = TRUE;
3227       if (!$this->_smartGroupCache || $group->cache_date == NULL) {
3228         CRM_Contact_BAO_GroupContactCache::load($group);
3229       }
3230     }
3231     if ($group->N == 0 && $op != 'NOT IN') {
3232       return NULL;
3233     }
3234
3235     $this->_tables[$tableAlias] = $this->_whereTables[$tableAlias] = " LEFT JOIN civicrm_group_contact_cache {$tableAlias} ON {$joinTable}.{$joinColumn} = {$tableAlias}.contact_id ";
3236
3237     if ($op == 'NOT IN') {
3238       return "{$tableAlias}.contact_id NOT IN (SELECT contact_id FROM civicrm_group_contact_cache cgcc WHERE cgcc.group_id IN ( " . implode(',', (array) $groupsIds) . " ) )";
3239     }
3240     return self::buildClause("{$tableAlias}.group_id", $op, $groups, 'Int');
3241   }
3242
3243   /**
3244    * Where / qill clause for cms users
3245    *
3246    * @param $values
3247    */
3248   public function ufUser(&$values) {
3249     list($name, $op, $value, $grouping, $wildcard) = $values;
3250
3251     if ($value == 1) {
3252       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' INNER JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3253
3254       $this->_qill[$grouping][] = ts('CMS User');
3255     }
3256     elseif ($value == 0) {
3257       $this->_tables['civicrm_uf_match'] = $this->_whereTables['civicrm_uf_match'] = ' LEFT JOIN civicrm_uf_match ON civicrm_uf_match.contact_id = contact_a.id ';
3258
3259       $this->_where[$grouping][] = " civicrm_uf_match.contact_id IS NULL";
3260       $this->_qill[$grouping][] = ts('Not a CMS User');
3261     }
3262   }
3263
3264   /**
3265    * All tag search specific.
3266    *
3267    * @param array $values
3268    */
3269   public function tagSearch(&$values) {
3270     list($name, $op, $value, $grouping, $wildcard) = $values;
3271
3272     $op = "LIKE";
3273     $value = "%{$value}%";
3274     $escapedValue = CRM_Utils_Type::escape("%{$value}%", 'String');
3275
3276     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3277     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3278
3279     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3280     $tTable = "`civicrm_tag-" . uniqid() . "`";
3281
3282     if ($useAllTagTypes[2]) {
3283       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3284         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id)
3285             LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  )";
3286
3287       // search tag in cases
3288       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3289       $tCaseTable = "`civicrm_case_tag-" . uniqid() . "`";
3290       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3291         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3292             LEFT JOIN civicrm_case
3293             ON (civicrm_case_contact.case_id = civicrm_case.id
3294                 AND civicrm_case.is_deleted = 0 )
3295             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id )
3296             LEFT JOIN civicrm_tag {$tCaseTable} ON ( {$etCaseTable}.tag_id = {$tCaseTable}.id  )";
3297       // search tag in activities
3298       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3299       $tActTable = "`civicrm_act_tag-" . uniqid() . "`";
3300       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3301       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3302
3303       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3304         = " LEFT JOIN civicrm_activity_contact
3305             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3306             LEFT JOIN civicrm_activity
3307             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3308             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3309             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id )
3310             LEFT JOIN civicrm_tag {$tActTable} ON ( {$etActTable}.tag_id = {$tActTable}.id  )";
3311
3312       $this->_where[$grouping][] = "({$tTable}.name $op '" . $escapedValue . "' OR {$tCaseTable}.name $op '" . $escapedValue . "' OR {$tActTable}.name $op '" . $escapedValue . "')";
3313       $this->_qill[$grouping][] = ts('Tag %1 %2', [1 => $tagTypesText[2], 2 => $op]) . ' ' . $value;
3314     }
3315     else {
3316       $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3317       $tTable = "`civicrm_tag-" . uniqid() . "`";
3318       $this->_tables[$etTable] = $this->_whereTables[$etTable] = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND
3319       {$etTable}.entity_table = 'civicrm_contact' )
3320                 LEFT JOIN civicrm_tag {$tTable} ON ( {$etTable}.tag_id = {$tTable}.id  ) ";
3321
3322       $this->_where[$grouping][] = self::buildClause("{$tTable}.name", $op, $value, 'String');
3323       $this->_qill[$grouping][] = ts('Tagged %1', [1 => $op]) . ' ' . $value;
3324     }
3325   }
3326
3327   /**
3328    * Where / qill clause for tag.
3329    *
3330    * @param array $values
3331    */
3332   public function tag(&$values) {
3333     list($name, $op, $value, $grouping, $wildcard) = $values;
3334
3335     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_EntityTag', "tag_id", $value, $op, ['onlyActive' => FALSE]);
3336     // API/Search Builder format array(operator => array(values))
3337     if (is_array($value)) {
3338       if (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
3339         $op = key($value);
3340         $value = $value[$op];
3341       }
3342       if (count($value) > 1) {
3343         $this->_useDistinct = TRUE;
3344       }
3345     }
3346
3347     // implode array, then remove all spaces
3348     $value = str_replace(' ', '', implode(',', (array) $value));
3349     if (!empty($value)) {
3350       $value = CRM_Utils_Type::validate(
3351         $value,
3352         'CommaSeparatedIntegers'
3353       );
3354     }
3355
3356     $useAllTagTypes = $this->getWhereValues('all_tag_types', $grouping);
3357     $tagTypesText = $this->getWhereValues('tag_types_text', $grouping);
3358
3359     $etTable = "`civicrm_entity_tag-" . uniqid() . "`";
3360
3361     if ($useAllTagTypes[2]) {
3362       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3363         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3364
3365       // search tag in cases
3366       $etCaseTable = "`civicrm_entity_case_tag-" . uniqid() . "`";
3367       $activityContacts = CRM_Activity_BAO_ActivityContact::buildOptions('record_type_id', 'validate');
3368       $targetID = CRM_Utils_Array::key('Activity Targets', $activityContacts);
3369
3370       $this->_tables[$etCaseTable] = $this->_whereTables[$etCaseTable]
3371         = " LEFT JOIN civicrm_case_contact ON civicrm_case_contact.contact_id = contact_a.id
3372             LEFT JOIN civicrm_case
3373             ON (civicrm_case_contact.case_id = civicrm_case.id
3374                 AND civicrm_case.is_deleted = 0 )
3375             LEFT JOIN civicrm_entity_tag {$etCaseTable} ON ( {$etCaseTable}.entity_table = 'civicrm_case' AND {$etCaseTable}.entity_id = civicrm_case.id ) ";
3376       // search tag in activities
3377       $etActTable = "`civicrm_entity_act_tag-" . uniqid() . "`";
3378       $this->_tables[$etActTable] = $this->_whereTables[$etActTable]
3379         = " LEFT JOIN civicrm_activity_contact
3380             ON ( civicrm_activity_contact.contact_id = contact_a.id AND civicrm_activity_contact.record_type_id = {$targetID} )
3381             LEFT JOIN civicrm_activity
3382             ON ( civicrm_activity.id = civicrm_activity_contact.activity_id
3383             AND civicrm_activity.is_deleted = 0 AND civicrm_activity.is_current_revision = 1 )
3384             LEFT JOIN civicrm_entity_tag as {$etActTable} ON ( {$etActTable}.entity_table = 'civicrm_activity' AND {$etActTable}.entity_id = civicrm_activity.id ) ";
3385
3386       // CRM-10338
3387       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3388         $this->_where[$grouping][] = "({$etTable}.tag_id $op OR {$etCaseTable}.tag_id $op OR {$etActTable}.tag_id $op)";
3389       }
3390       else {
3391         $this->_where[$grouping][] = "({$etTable}.tag_id $op (" . $value . ") OR {$etCaseTable}.tag_id $op (" . $value . ") OR {$etActTable}.tag_id $op (" . $value . "))";
3392       }
3393     }
3394     else {
3395       $this->_tables[$etTable] = $this->_whereTables[$etTable]
3396         = " LEFT JOIN civicrm_entity_tag {$etTable} ON ( {$etTable}.entity_id = contact_a.id  AND {$etTable}.entity_table = 'civicrm_contact') ";
3397
3398       // CRM-10338
3399       if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3400         // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3401         $op = str_replace('EMPTY', 'NULL', $op);
3402         $this->_where[$grouping][] = "{$etTable}.tag_id $op";
3403       }
3404       // CRM-16941: for tag tried with != operator we don't show contact who don't have given $value AND also in other tag
3405       elseif ($op == '!=') {
3406         $this->_where[$grouping][] = "{$etTable}.entity_id NOT IN (SELECT entity_id FROM civicrm_entity_tag cet WHERE cet.entity_table = 'civicrm_contact' AND " . self::buildClause("cet.tag_id", '=', $value, 'Int') . ")";
3407       }
3408       elseif ($op == '=' || strstr($op, 'IN')) {
3409         $op = ($op == '=') ? 'IN' : $op;
3410         $this->_where[$grouping][] = "{$etTable}.tag_id $op ( $value )";
3411       }
3412     }
3413     $this->_qill[$grouping][] = ts('Tagged %1 %2', [1 => $qillop, 2 => $qillVal]);
3414   }
3415
3416   /**
3417    * Where/qill clause for notes
3418    *
3419    * @param array $values
3420    */
3421   public function notes(&$values) {
3422     list($name, $op, $value, $grouping, $wildcard) = $values;
3423
3424     $noteOptionValues = $this->getWhereValues('note_option', $grouping);
3425     $noteOption = CRM_Utils_Array::value('2', $noteOptionValues, '6');
3426     $noteOption = ($name == 'note_body') ? 2 : (($name == 'note_subject') ? 3 : $noteOption);
3427
3428     $this->_useDistinct = TRUE;
3429
3430     $this->_tables['civicrm_note'] = $this->_whereTables['civicrm_note']
3431       = " LEFT JOIN civicrm_note ON ( civicrm_note.entity_table = 'civicrm_contact' AND contact_a.id = civicrm_note.entity_id ) ";
3432
3433     $n = trim($value);
3434     $value = CRM_Core_DAO::escapeString($n);
3435     if ($wildcard) {
3436       if (strpos($value, '%') === FALSE) {
3437         $value = "%$value%";
3438       }
3439       $op = 'LIKE';
3440     }
3441     elseif ($op == 'IS NULL' || $op == 'IS NOT NULL') {
3442       $value = NULL;
3443     }
3444
3445     $label = NULL;
3446     $clauses = [];
3447     if ($noteOption % 2 == 0) {
3448       $clauses[] = self::buildClause('civicrm_note.note', $op, $value, 'String');
3449       $label = ts('Note: Body Only');
3450     }
3451     if ($noteOption % 3 == 0) {
3452       $clauses[] = self::buildClause('civicrm_note.subject', $op, $value, 'String');
3453       $label = $label ? ts('Note: Body and Subject') : ts('Note: Subject Only');
3454     }
3455     $this->_where[$grouping][] = "( " . implode(' OR ', $clauses) . " )";
3456     list($qillOp, $qillVal) = self::buildQillForFieldValue(NULL, $name, $n, $op);
3457     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillOp, 3 => $qillVal]);
3458   }
3459
3460   /**
3461    * @param string $name
3462    * @param $op
3463    * @param $grouping
3464    *
3465    * @return bool
3466    */
3467   public function nameNullOrEmptyOp($name, $op, $grouping) {
3468     switch ($op) {
3469       case 'IS NULL':
3470       case 'IS NOT NULL':
3471         $this->_where[$grouping][] = "contact_a.$name $op";
3472         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3473         return TRUE;
3474
3475       case 'IS EMPTY':
3476         $this->_where[$grouping][] = "(contact_a.$name IS NULL OR contact_a.$name = '')";
3477         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3478         return TRUE;
3479
3480       case 'IS NOT EMPTY':
3481         $this->_where[$grouping][] = "(contact_a.$name IS NOT NULL AND contact_a.$name <> '')";
3482         $this->_qill[$grouping][] = ts('Name') . ' ' . $op;
3483         return TRUE;
3484
3485       default:
3486         return FALSE;
3487     }
3488   }
3489
3490   /**
3491    * Where / qill clause for sort_name
3492    *
3493    * @param array $values
3494    */
3495   public function sortName(&$values) {
3496     list($fieldName, $op, $value, $grouping, $wildcard) = $values;
3497
3498     // handle IS NULL / IS NOT NULL / IS EMPTY / IS NOT EMPTY
3499     if ($this->nameNullOrEmptyOp($fieldName, $op, $grouping)) {
3500       return;
3501     }
3502
3503     $input = $value = is_array($value) ? trim($value['LIKE']) : trim($value);
3504
3505     if (!strlen($value)) {
3506       return;
3507     }
3508
3509     $config = CRM_Core_Config::singleton();
3510
3511     $sub = [];
3512
3513     //By default, $sub elements should be joined together with OR statements (don't change this variable).
3514     $subGlue = ' OR ';
3515
3516     $firstChar = substr($value, 0, 1);
3517     $lastChar = substr($value, -1, 1);
3518     $quotes = ["'", '"'];
3519     // If string is quoted, strip quotes and otherwise don't alter it
3520     if ((strlen($value) > 2) && in_array($firstChar, $quotes) && in_array($lastChar, $quotes)) {
3521       $value = trim($value, implode('', $quotes));
3522     }
3523     // Replace spaces with wildcards for a LIKE operation
3524     // UNLESS string contains a comma (this exception is a tiny bit questionable)
3525     elseif ($op == 'LIKE' && strpos($value, ',') === FALSE) {
3526       $value = str_replace(' ', '%', $value);
3527     }
3528     $value = CRM_Core_DAO::escapeString(trim($value));
3529     if (strlen($value)) {
3530       $fieldsub = [];
3531       $value = "'" . self::getWildCardedValue($wildcard, $op, $value) . "'";
3532       if ($fieldName == 'sort_name') {
3533         $wc = "contact_a.sort_name";
3534       }
3535       else {
3536         $wc = "contact_a.display_name";
3537       }
3538       $fieldsub[] = " ( $wc $op $value )";
3539       if ($config->includeNickNameInName) {
3540         $wc = "contact_a.nick_name";
3541         $fieldsub[] = " ( $wc $op $value )";
3542       }
3543       if ($config->includeEmailInName) {
3544         $fieldsub[] = " ( civicrm_email.email $op $value ) ";
3545       }
3546       $sub[] = ' ( ' . implode(' OR ', $fieldsub) . ' ) ';
3547     }
3548
3549     $sub = ' ( ' . implode($subGlue, $sub) . ' ) ';
3550
3551     $this->_where[$grouping][] = $sub;
3552     if ($config->includeEmailInName) {
3553       $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3554       $this->_qill[$grouping][] = ts('Name or Email') . " $op - '$input'";
3555     }
3556     else {
3557       $this->_qill[$grouping][] = ts('Name') . " $op - '$input'";
3558     }
3559   }
3560
3561   /**
3562    * Where/qill clause for greeting fields.
3563    *
3564    * @param array $values
3565    */
3566   public function greetings(&$values) {
3567     list($name, $op, $value, $grouping, $wildcard) = $values;
3568     $name .= '_display';
3569
3570     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
3571     $this->_qill[$grouping][] = ts('Greeting %1 %2', [1 => $qillop, 2 => $qillVal]);
3572     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value, 'String');
3573   }
3574
3575   /**
3576    * Where / qill clause for email
3577    *
3578    * @param array $values
3579    * @param string $isForcePrimaryOnly
3580    *
3581    * @throws \CRM_Core_Exception
3582    */
3583   protected function email(&$values, $isForcePrimaryOnly) {
3584     list($name, $op, $value, $grouping, $wildcard) = $values;
3585     $this->_tables['civicrm_email'] = $this->_whereTables['civicrm_email'] = 1;
3586
3587     // CRM-18147: for Contact's GET API, email fieldname got appended with its entity as in {$apiEntiy}_{$name}
3588     // so following code is use build whereClause for contact's primart email id
3589     if (!empty($isForcePrimaryOnly)) {
3590       $this->_where[$grouping][] = self::buildClause('civicrm_email.is_primary', '=', 1, 'Integer');
3591     }
3592     // @todo - this should come from the $this->_fields array
3593     $dbName = $name === 'email_id' ? 'id' : $name;
3594
3595     if (is_array($value) || $name === 'email_id') {
3596       $this->_qill[$grouping][] = $this->getQillForField($name, $value, $op);
3597       $this->_where[$grouping][] = self::buildClause('civicrm_email.' . $dbName, $op, $value, 'String');
3598       return;
3599     }
3600
3601     // Is this ever hit now? Ideally ensure always an array & handle above.
3602     $n = trim($value);
3603     if ($n) {
3604       if (substr($n, 0, 1) == '"' &&
3605         substr($n, -1, 1) == '"'
3606       ) {
3607         $n = substr($n, 1, -1);
3608         $value = CRM_Core_DAO::escapeString($n);
3609         $op = '=';
3610       }
3611       else {
3612         $value = self::getWildCardedValue($wildcard, $op, $n);
3613       }
3614       $this->_qill[$grouping][] = ts('Email') . " $op '$n'";
3615       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, $value, 'String');
3616     }
3617     else {
3618       $this->_qill[$grouping][] = ts('Email') . " $op ";
3619       $this->_where[$grouping][] = self::buildClause('civicrm_email.email', $op, NULL, 'String');
3620     }
3621   }
3622
3623   /**
3624    * Where / qill clause for phone number
3625    *
3626    * @param array $values
3627    */
3628   public function phone_numeric(&$values) {
3629     list($name, $op, $value, $grouping, $wildcard) = $values;
3630     // Strip non-numeric characters; allow wildcards
3631     $number = preg_replace('/[^\d%]/', '', $value);
3632     if ($number) {
3633       if (strpos($number, '%') === FALSE) {
3634         $number = "%$number%";
3635       }
3636
3637       $this->_qill[$grouping][] = ts('Phone number contains') . " $number";
3638       $this->_where[$grouping][] = self::buildClause('civicrm_phone.phone_numeric', 'LIKE', "$number", 'String');
3639       $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3640     }
3641   }
3642
3643   /**
3644    * Where / qill clause for phone type/location
3645    *
3646    * @param array $values
3647    */
3648   public function phone_option_group($values) {
3649     list($name, $op, $value, $grouping, $wildcard) = $values;
3650     $option = ($name == 'phone_phone_type_id' ? 'phone_type_id' : 'location_type_id');
3651     $options = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', $option);
3652     $optionName = $options[$value];
3653     $this->_qill[$grouping][] = ts('Phone') . ' ' . ($name == 'phone_phone_type_id' ? ts('type') : ('location')) . " $op $optionName";
3654     $this->_where[$grouping][] = self::buildClause('civicrm_phone.' . substr($name, 6), $op, $value, 'Integer');
3655     $this->_tables['civicrm_phone'] = $this->_whereTables['civicrm_phone'] = 1;
3656   }
3657
3658   /**
3659    * Where / qill clause for street_address.
3660    *
3661    * @param array $values
3662    */
3663   public function street_address(&$values) {
3664     list($name, $op, $value, $grouping) = $values;
3665
3666     if (!$op) {
3667       $op = 'LIKE';
3668     }
3669
3670     $n = trim($value);
3671
3672     if ($n) {
3673       if (strpos($value, '%') === FALSE) {
3674         // only add wild card if not there
3675         $value = "%{$value}%";
3676       }
3677       $op = 'LIKE';
3678       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, $value, 'String');
3679       $this->_qill[$grouping][] = ts('Street') . " $op '$n'";
3680     }
3681     else {
3682       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_address', $op, NULL, 'String');
3683       $this->_qill[$grouping][] = ts('Street') . " $op ";
3684     }
3685
3686     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3687   }
3688
3689   /**
3690    * Where / qill clause for street_unit.
3691    *
3692    * @param array $values
3693    */
3694   public function street_number(&$values) {
3695     list($name, $op, $value, $grouping, $wildcard) = $values;
3696
3697     if (!$op) {
3698       $op = '=';
3699     }
3700
3701     $n = trim($value);
3702
3703     if (strtolower($n) == 'odd') {
3704       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 1 )";
3705       $this->_qill[$grouping][] = ts('Street Number is odd');
3706     }
3707     elseif (strtolower($n) == 'even') {
3708       $this->_where[$grouping][] = " ( civicrm_address.street_number % 2 = 0 )";
3709       $this->_qill[$grouping][] = ts('Street Number is even');
3710     }
3711     else {
3712       $value = $n;
3713       $this->_where[$grouping][] = self::buildClause('civicrm_address.street_number', $op, $value, 'String');
3714       $this->_qill[$grouping][] = ts('Street Number') . " $op '$n'";
3715     }
3716
3717     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3718   }
3719
3720   /**
3721    * Where / qill clause for sorting by character.
3722    *
3723    * @param array $values
3724    */
3725   public function sortByCharacter(&$values) {
3726     list($name, $op, $value, $grouping, $wildcard) = $values;
3727
3728     $name = trim($value);
3729     $cond = " contact_a.sort_name LIKE '" . CRM_Core_DAO::escapeWildCardString($name) . "%'";
3730     $this->_where[$grouping][] = $cond;
3731     $this->_qill[$grouping][] = ts('Showing only Contacts starting with: \'%1\'', [1 => $name]);
3732   }
3733
3734   /**
3735    * Where / qill clause for including contact ids.
3736    */
3737   public function includeContactIDs() {
3738     if (!$this->_includeContactIds || empty($this->_params)) {
3739       return;
3740     }
3741
3742     $contactIds = [];
3743     foreach ($this->_params as $id => $values) {
3744       if (substr($values[0], 0, CRM_Core_Form::CB_PREFIX_LEN) == CRM_Core_Form::CB_PREFIX) {
3745         $contactIds[] = substr($values[0], CRM_Core_Form::CB_PREFIX_LEN);
3746       }
3747     }
3748     CRM_Utils_Type::validateAll($contactIds, 'Positive');
3749     if (!empty($contactIds)) {
3750       $this->_where[0][] = " ( contact_a.id IN (" . implode(',', $contactIds) . " ) ) ";
3751     }
3752   }
3753
3754   /**
3755    * Where / qill clause for postal code.
3756    *
3757    * @param array $values
3758    */
3759   public function postalCode(&$values) {
3760     // skip if the fields dont have anything to do with postal_code
3761     if (empty($this->_fields['postal_code'])) {
3762       return;
3763     }
3764
3765     list($name, $op, $value, $grouping, $wildcard) = $values;
3766
3767     // Handle numeric postal code range searches properly by casting the column as numeric
3768     if (is_numeric($value)) {
3769       $field = "IF (civicrm_address.postal_code REGEXP '^[0-9]{1,10}$', CAST(civicrm_address.postal_code AS UNSIGNED), 0)";
3770       $val = CRM_Utils_Type::escape($value, 'Integer');
3771     }
3772     else {
3773       $field = 'civicrm_address.postal_code';
3774       // Per CRM-17060 we might be looking at an 'IN' syntax so don't case arrays to string.
3775       if (!is_array($value)) {
3776         $val = CRM_Utils_Type::escape($value, 'String');
3777       }
3778       else {
3779         // Do we need to escape values here? I would expect buildClause does.
3780         $val = $value;
3781       }
3782     }
3783
3784     $this->_tables['civicrm_address'] = $this->_whereTables['civicrm_address'] = 1;
3785
3786     if ($name == 'postal_code') {
3787       $this->_where[$grouping][] = self::buildClause($field, $op, $val, 'String');
3788       $this->_qill[$grouping][] = ts('Postal code') . " {$op} {$value}";
3789     }
3790     elseif ($name == 'postal_code_low') {
3791       $this->_where[$grouping][] = " ( $field >= '$val' ) ";
3792       $this->_qill[$grouping][] = ts('Postal code greater than or equal to \'%1\'', [1 => $value]);
3793     }
3794     elseif ($name == 'postal_code_high') {
3795       $this->_where[$grouping][] = " ( $field <= '$val' ) ";
3796       $this->_qill[$grouping][] = ts('Postal code less than or equal to \'%1\'', [1 => $value]);
3797     }
3798   }
3799
3800   /**
3801    * Where / qill clause for location type.
3802    *
3803    * @param array $values
3804    * @param null $status
3805    *
3806    * @return string
3807    */
3808   public function locationType(&$values, $status = NULL) {
3809     list($name, $op, $value, $grouping, $wildcard) = $values;
3810
3811     if (is_array($value)) {
3812       $this->_where[$grouping][] = 'civicrm_address.location_type_id IN (' . implode(',', $value) . ')';
3813       $this->_tables['civicrm_address'] = 1;
3814       $this->_whereTables['civicrm_address'] = 1;
3815
3816       $locationType = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
3817       $names = [];
3818       foreach ($value as $id) {
3819         $names[] = $locationType[$id];
3820       }
3821
3822       $this->_primaryLocation = FALSE;
3823
3824       if (!$status) {
3825         $this->_qill[$grouping][] = ts('Location Type') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3826       }
3827       else {
3828         return implode(' ' . ts('or') . ' ', $names);
3829       }
3830     }
3831   }
3832
3833   /**
3834    * @param $values
3835    * @param bool $fromStateProvince
3836    *
3837    * @return array|NULL
3838    */
3839   public function country(&$values, $fromStateProvince = TRUE) {
3840     list($name, $op, $value, $grouping, $wildcard) = $values;
3841
3842     if (!$fromStateProvince) {
3843       $stateValues = $this->getWhereValues('state_province', $grouping);
3844       if (!empty($stateValues)) {
3845         // return back to caller if there are state province values
3846         // since that handles this case
3847         return NULL;
3848       }
3849     }
3850
3851     $countryClause = $countryQill = NULL;
3852     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) || ($values && !empty($value))) {
3853       $this->_tables['civicrm_address'] = 1;
3854       $this->_whereTables['civicrm_address'] = 1;
3855
3856       $countryClause = self::buildClause('civicrm_address.country_id', $op, $value, 'Positive');
3857       list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, 'country_id', $value, $op);
3858       $countryQill = ts("%1 %2 %3", [1 => 'Country', 2 => $qillop, 3 => $qillVal]);
3859
3860       if (!$fromStateProvince) {
3861         $this->_where[$grouping][] = $countryClause;
3862         $this->_qill[$grouping][] = $countryQill;
3863       }
3864     }
3865
3866     if ($fromStateProvince) {
3867       if (!empty($countryClause)) {
3868         return [
3869           $countryClause,
3870           " ...AND... " . $countryQill,
3871         ];
3872       }
3873       else {
3874         return [NULL, NULL];
3875       }
3876     }
3877   }
3878
3879   /**
3880    * Where / qill clause for county (if present).
3881    *
3882    * @param array $values
3883    * @param null $status
3884    *
3885    * @return string
3886    */
3887   public function county(&$values, $status = NULL) {
3888     list($name, $op, $value, $grouping, $wildcard) = $values;
3889
3890     if (!is_array($value)) {
3891       // force the county to be an array
3892       $value = [$value];
3893     }
3894
3895     // check if the values are ids OR names of the counties
3896     $inputFormat = 'id';
3897     foreach ($value as $v) {
3898       if (!is_numeric($v)) {
3899         $inputFormat = 'name';
3900         break;
3901       }
3902     }
3903     $names = [];
3904     if ($op == '=') {
3905       $op = 'IN';
3906     }
3907     elseif ($op == '!=') {
3908       $op = 'NOT IN';
3909     }
3910     else {
3911       // this converts IS (NOT)? EMPTY to IS (NOT)? NULL
3912       $op = str_replace('EMPTY', 'NULL', $op);
3913     }
3914
3915     if (in_array($op, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
3916       $clause = "civicrm_address.county_id $op";
3917     }
3918     elseif ($inputFormat == 'id') {
3919       $clause = 'civicrm_address.county_id IN (' . implode(',', $value) . ')';
3920
3921       $county = CRM_Core_PseudoConstant::county();
3922       foreach ($value as $id) {
3923         $names[] = CRM_Utils_Array::value($id, $county);
3924       }
3925     }
3926     else {
3927       $inputClause = [];
3928       $county = CRM_Core_PseudoConstant::county();
3929       foreach ($value as $name) {
3930         $name = trim($name);
3931         $inputClause[] = CRM_Utils_Array::key($name, $county);
3932       }
3933       $clause = 'civicrm_address.county_id IN (' . implode(',', $inputClause) . ')';
3934       $names = $value;
3935     }
3936     $this->_tables['civicrm_address'] = 1;
3937     $this->_whereTables['civicrm_address'] = 1;
3938
3939     $this->_where[$grouping][] = $clause;
3940     if (!$status) {
3941       $this->_qill[$grouping][] = ts('County') . ' - ' . implode(' ' . ts('or') . ' ', $names);
3942     }
3943     else {
3944       return implode(' ' . ts('or') . ' ', $names);
3945     }
3946   }
3947
3948   /**
3949    * Where / qill clause for state/province AND country (if present).
3950    *
3951    * @param array $values
3952    * @param null $status
3953    *
3954    * @return string
3955    */
3956   public function stateProvince(&$values, $status = NULL) {
3957     list($name, $op, $value, $grouping, $wildcard) = $values;
3958
3959     $stateClause = self::buildClause('civicrm_address.state_province_id', $op, $value, 'Positive');
3960     $this->_tables['civicrm_address'] = 1;
3961     $this->_whereTables['civicrm_address'] = 1;
3962
3963     $countryValues = $this->getWhereValues('country', $grouping);
3964     list($countryClause, $countryQill) = $this->country($countryValues, TRUE);
3965     if ($countryClause) {
3966       $clause = "( $stateClause AND $countryClause )";
3967     }
3968     else {
3969       $clause = $stateClause;
3970     }
3971
3972     $this->_where[$grouping][] = $clause;
3973     list($qillop, $qillVal) = self::buildQillForFieldValue('CRM_Core_DAO_Address', "state_province_id", $value, $op);
3974     if (!$status) {
3975       $this->_qill[$grouping][] = ts("State/Province %1 %2 %3", [1 => $qillop, 2 => $qillVal, 3 => $countryQill]);
3976     }
3977     else {
3978       return implode(' ' . ts('or') . ' ', $qillVal) . $countryQill;
3979     }
3980   }
3981
3982   /**
3983    * Where / qill clause for change log.
3984    *
3985    * @param array $values
3986    */
3987   public function changeLog(&$values) {
3988     list($name, $op, $value, $grouping, $wildcard) = $values;
3989
3990     $targetName = $this->getWhereValues('changed_by', $grouping);
3991     if (!$targetName) {
3992       return;
3993     }
3994
3995     $name = trim($targetName[2]);
3996     $name = CRM_Core_DAO::escapeString($name);
3997     $name = $targetName[4] ? "%$name%" : $name;
3998     $this->_where[$grouping][] = "contact_b_log.sort_name LIKE '%$name%'";
3999     $this->_tables['civicrm_log'] = $this->_whereTables['civicrm_log'] = 1;
4000     $fieldTitle = ts('Added By');
4001     foreach ($this->_params as $params) {
4002       if ($params[0] == 'log_date') {
4003         if ($params[2] == 2) {
4004           $fieldTitle = ts('Modified By');
4005         }
4006         break;
4007       }
4008     }
4009     list($qillop, $qillVal) = self::buildQillForFieldValue(NULL, 'changed_by', $name, 'LIKE');
4010     $this->_qill[$grouping][] = ts("%1 %2 '%3'", [
4011       1 => $fieldTitle,
4012       2 => $qillop,
4013       3 => $qillVal,
4014     ]);
4015   }
4016
4017   /**
4018    * @param $values
4019    */
4020   public function modifiedDates($values) {
4021     $this->_useDistinct = TRUE;
4022
4023     // CRM-11281, default to added date if not set
4024     $fieldTitle = ts('Added Date');
4025     $fieldName = 'created_date';
4026     foreach (array_keys($this->_params) as $id) {
4027       if ($this->_params[$id][0] == 'log_date') {
4028         if ($this->_params[$id][2] == 2) {
4029           $fieldTitle = ts('Modified Date');
4030           $fieldName = 'modified_date';
4031         }
4032       }
4033     }
4034
4035     $this->dateQueryBuilder($values, 'contact_a', 'log_date', $fieldName, $fieldTitle);
4036
4037     self::$_openedPanes[ts('Change Log')] = TRUE;
4038   }
4039
4040   /**
4041    * @param $values
4042    */
4043   public function demographics(&$values) {
4044     list($name, $op, $value, $grouping, $wildcard) = $values;
4045
4046     if (($name == 'age_low') || ($name == 'age_high')) {
4047       $this->ageRangeQueryBuilder($values,
4048         'contact_a', 'age', 'birth_date', ts('Age')
4049       );
4050     }
4051     elseif (($name == 'birth_date_low') || ($name == 'birth_date_high')) {
4052
4053       $this->dateQueryBuilder($values,
4054         'contact_a', 'birth_date', 'birth_date', ts('Birth Date')
4055       );
4056     }
4057     elseif (($name == 'deceased_date_low') || ($name == 'deceased_date_high')) {
4058
4059       $this->dateQueryBuilder($values,
4060         'contact_a', 'deceased_date', 'deceased_date', ts('Deceased Date')
4061       );
4062     }
4063
4064     self::$_openedPanes[ts('Demographics')] = TRUE;
4065   }
4066
4067   /**
4068    * @param $values
4069    */
4070   public function privacy(&$values) {
4071     list($name, $op, $value, $grouping) = $values;
4072     //fixed for profile search listing CRM-4633
4073     if (strpbrk($value, "[")) {
4074       $value = "'{$value}'";
4075       $op = "!{$op}";
4076       $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4077     }
4078     else {
4079       $this->_where[$grouping][] = "contact_a.{$name} $op $value";
4080     }
4081     $field = CRM_Utils_Array::value($name, $this->_fields);
4082     $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
4083     $title = $field ? $field['title'] : $name;
4084     $this->_qill[$grouping][] = "$title $op $value";
4085   }
4086
4087   /**
4088    * @param $values
4089    */
4090   public function privacyOptions($values) {
4091     list($name, $op, $value, $grouping, $wildcard) = $values;
4092
4093     if (empty($value) || !is_array($value)) {
4094       return;
4095     }
4096
4097     // get the operator and toggle values
4098     $opValues = $this->getWhereValues('privacy_operator', $grouping);
4099     $operator = 'OR';
4100     if ($opValues &&
4101       strtolower($opValues[2] == 'AND')
4102     ) {
4103       // @todo this line is logially unreachable
4104       $operator = 'AND';
4105     }
4106
4107     $toggleValues = $this->getWhereValues('privacy_toggle', $grouping);
4108     $compareOP = '!';
4109     if ($toggleValues &&
4110       $toggleValues[2] == 2
4111     ) {
4112       $compareOP = '';
4113     }
4114
4115     $clauses = [];
4116     $qill = [];
4117     foreach ($value as $dontCare => $pOption) {
4118       $clauses[] = " ( contact_a.{$pOption} = 1 ) ";
4119       $field = CRM_Utils_Array::value($pOption, $this->_fields);
4120       $title = $field ? $field['title'] : $pOption;
4121       $qill[] = " $title = 1 ";
4122     }
4123
4124     $this->_where[$grouping][] = $compareOP . '( ' . implode($operator, $clauses) . ' )';
4125     $this->_qill[$grouping][] = $compareOP . '( ' . implode($operator, $qill) . ' )';
4126   }
4127
4128   /**
4129    * @param $values
4130    */
4131   public function preferredCommunication(&$values) {
4132     list($name, $op, $value, $grouping, $wildcard) = $values;
4133
4134     if (!is_array($value)) {
4135       $value = str_replace(['(', ')'], '', explode(",", $value));
4136     }
4137     elseif (in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
4138       $op = key($value);
4139       $value = $value[$op];
4140     }
4141     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue('CRM_Contact_DAO_Contact', $name, $value, $op);
4142
4143     if (self::caseImportant($op)) {
4144       $value = implode("[[:cntrl:]]|[[:cntrl:]]", (array) $value);
4145       $op = (strstr($op, '!') || strstr($op, 'NOT')) ? 'NOT RLIKE' : 'RLIKE';
4146       $value = "[[:cntrl:]]" . $value . "[[:cntrl:]]";
4147     }
4148
4149     $this->_where[$grouping][] = self::buildClause("contact_a.preferred_communication_method", $op, $value);
4150     $this->_qill[$grouping][] = ts('Preferred Communication Method %1 %2', [1 => $qillop, 2 => $qillVal]);
4151   }
4152
4153   /**
4154    * Where / qill clause for relationship.
4155    *
4156    * @param array $values
4157    */
4158   public function relationship(&$values) {
4159     list($name, $op, $value, $grouping, $wildcard) = $values;
4160     if ($this->_relationshipValuesAdded) {
4161       return;
4162     }
4163     // also get values array for relation_target_name
4164     // for relationship search we always do wildcard
4165     $relationType = $this->getWhereValues('relation_type_id', $grouping);
4166     $description = $this->getWhereValues('relation_description', $grouping);
4167     $targetName = $this->getWhereValues('relation_target_name', $grouping);
4168     $relStatus = $this->getWhereValues('relation_status', $grouping);
4169     $targetGroup = $this->getWhereValues('relation_target_group', $grouping);
4170
4171     $nameClause = $name = NULL;
4172     if ($targetName) {
4173       $name = trim($targetName[2]);
4174       if (substr($name, 0, 1) == '"' &&
4175         substr($name, -1, 1) == '"'
4176       ) {
4177         $name = substr($name, 1, -1);
4178         $name = CRM_Core_DAO::escapeString($name);
4179         $nameClause = "= '$name'";
4180       }
4181       else {
4182         $name = CRM_Core_DAO::escapeString($name);
4183         $nameClause = "LIKE '%{$name}%'";
4184       }
4185     }
4186
4187     $relTypes = $relTypesIds = [];
4188     if (!empty($relationType)) {
4189       $relationType[2] = (array) $relationType[2];
4190       foreach ($relationType[2] as $relType) {
4191         $rel = explode('_', $relType);
4192         self::$_relType = $rel[1];
4193         $params = ['id' => $rel[0]];
4194         $typeValues = [];
4195         $rTypeValue = CRM_Contact_BAO_RelationshipType::retrieve($params, $typeValues);
4196         if (!empty($rTypeValue)) {
4197           if ($rTypeValue->name_a_b == $rTypeValue->name_b_a) {
4198             // if we don't know which end of the relationship we are dealing with we'll create a temp table
4199             self::$_relType = 'reciprocal';
4200           }
4201           $relTypesIds[] = $rel[0];
4202           $relTypes[] = $relType;
4203         }
4204       }
4205     }
4206
4207     // if we are creating a temp table we build our own where for the relationship table
4208     $relationshipTempTable = NULL;
4209     if (self::$_relType == 'reciprocal') {
4210       $where = [];
4211       self::$_relationshipTempTable = $relationshipTempTable = CRM_Core_DAO::createTempTableName('civicrm_rel');
4212       if ($nameClause) {
4213         $where[$grouping][] = " sort_name $nameClause ";
4214       }
4215       $groupJoinTable = "civicrm_relationship";
4216       $groupJoinColumn = "contact_id_alt";
4217     }
4218     else {
4219       $where = &$this->_where;
4220       if ($nameClause) {
4221         $where[$grouping][] = "( contact_b.sort_name $nameClause AND contact_b.id != contact_a.id )";
4222       }
4223       $groupJoinTable = "contact_b";
4224       $groupJoinColumn = "id";
4225     }
4226     $allRelationshipType = CRM_Contact_BAO_Relationship::getContactRelationshipType(NULL, 'null', NULL, NULL, TRUE);
4227     if ($nameClause || !$targetGroup) {
4228       if (!empty($relationType)) {
4229         $relQill = '';
4230         foreach ($relTypes as $rel) {
4231           if (!empty($relQill)) {
4232             $relQill .= ' OR ';
4233           }
4234           $relQill .= $allRelationshipType[$rel];
4235         }
4236         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " $name";
4237       }
4238       else {
4239         $this->_qill[$grouping][] = $name;
4240       }
4241     }
4242
4243     //check to see if the target contact is in specified group
4244     if ($targetGroup) {
4245       //add contacts from static groups
4246       $this->_tables['civicrm_relationship_group_contact'] = $this->_whereTables['civicrm_relationship_group_contact']
4247         = " LEFT JOIN civicrm_group_contact civicrm_relationship_group_contact ON civicrm_relationship_group_contact.contact_id = {$groupJoinTable}.{$groupJoinColumn} AND civicrm_relationship_group_contact.status = 'Added'";
4248       $groupWhere[] = "( civicrm_relationship_group_contact.group_id IN  (" .
4249         implode(",", $targetGroup[2]) . ") ) ";
4250
4251       //add contacts from saved searches
4252       $ssWhere = $this->addGroupContactCache($targetGroup[2], "civicrm_relationship_group_contact_cache", $groupJoinTable, $op, $groupJoinColumn);
4253
4254       //set the group where clause
4255       if ($ssWhere) {
4256         $groupWhere[] = "( " . $ssWhere . " )";
4257       }
4258       $this->_where[$grouping][] = "( " . implode(" OR ", $groupWhere) . " )";
4259
4260       //Get the names of the target groups for the qill
4261       $groupNames = CRM_Core_PseudoConstant::group();
4262       $qillNames = [];
4263       foreach ($targetGroup[2] as $groupId) {
4264         if (array_key_exists($groupId, $groupNames)) {
4265           $qillNames[] = $groupNames[$groupId];
4266         }
4267       }
4268       if (!empty($relationType)) {
4269         $relQill = '';
4270         foreach ($relTypes as $rel) {
4271           if (!empty($relQill)) {
4272             $relQill .= ' OR ';
4273           }
4274           $relQill .= CRM_Utils_Array::value($rel, $allRelationshipType);
4275         }
4276         $this->_qill[$grouping][] = 'Relationship Type(s) ' . $relQill . " ( " . implode(", ", $qillNames) . " )";
4277       }
4278       else {
4279         $this->_qill[$grouping][] = implode(", ", $qillNames);
4280       }
4281     }
4282
4283     // Description
4284     if (!empty($description[2]) && trim($description[2])) {
4285       $this->_qill[$grouping][] = ts('Relationship description - ' . $description[2]);
4286       $description = CRM_Core_DAO::escapeString(trim($description[2]));
4287       $where[$grouping][] = "civicrm_relationship.description LIKE '%{$description}%'";
4288     }
4289
4290     // Note we do not currently set mySql to handle timezones, so doing this the old-fashioned way
4291     $today = date('Ymd');
4292     //check for active, inactive and all relation status
4293     if ($relStatus[2] == 0) {
4294       $where[$grouping][] = "(
4295 civicrm_relationship.is_active = 1 AND
4296 ( civicrm_relationship.end_date IS NULL OR civicrm_relationship.end_date >= {$today} ) AND
4297 ( civicrm_relationship.start_date IS NULL OR civicrm_relationship.start_date <= {$today} )
4298 )";
4299       $this->_qill[$grouping][] = ts('Relationship - Active and Current');
4300     }
4301     elseif ($relStatus[2] == 1) {
4302       $where[$grouping][] = "(
4303 civicrm_relationship.is_active = 0 OR
4304 civicrm_relationship.end_date < {$today} OR
4305 civicrm_relationship.start_date > {$today}
4306 )";
4307       $this->_qill[$grouping][] = ts('Relationship - Inactive or not Current');
4308     }
4309
4310     $onlyDeleted = 0;
4311     if (in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params)) {
4312       $onlyDeleted = 1;
4313     }
4314     $where[$grouping][] = "(contact_b.is_deleted = {$onlyDeleted})";
4315
4316     $this->addRelationshipPermissionClauses($grouping, $where);
4317     $this->addRelationshipDateClauses($grouping, $where);
4318     $this->addRelationshipActivePeriodClauses($grouping, $where);
4319     if (!empty($relTypes)) {
4320       $where[$grouping][] = 'civicrm_relationship.relationship_type_id IN (' . implode(',', $relTypesIds) . ')';
4321     }
4322     $this->_tables['civicrm_relationship'] = $this->_whereTables['civicrm_relationship'] = 1;
4323     $this->_useDistinct = TRUE;
4324     $this->_relationshipValuesAdded = TRUE;
4325     // it could be a or b, using an OR creates an unindexed join - better to create a temp table &
4326     // join on that,
4327     if ($relationshipTempTable) {
4328       $whereClause = '';
4329       if (!empty($where[$grouping])) {
4330         $whereClause = ' WHERE ' . implode(' AND ', $where[$grouping]);
4331         $whereClause = str_replace('contact_b', 'c', $whereClause);
4332       }
4333       $sql = "
4334         CREATE TEMPORARY TABLE {$relationshipTempTable}
4335           (
4336             `contact_id` int(10) unsigned NOT NULL DEFAULT '0',
4337             `contact_id_alt` int(10) unsigned NOT NULL DEFAULT '0',
4338             KEY `contact_id` (`contact_id`),
4339             KEY `contact_id_alt` (`contact_id_alt`)
4340           )
4341           (SELECT contact_id_b as contact_id, contact_id_a as contact_id_alt, civicrm_relationship.id
4342             FROM civicrm_relationship
4343             INNER JOIN  civicrm_contact c ON civicrm_relationship.contact_id_a = c.id
4344             $whereClause )
4345           UNION
4346             (SELECT contact_id_a as contact_id, contact_id_b as contact_id_alt, civicrm_relationship.id
4347             FROM civicrm_relationship
4348             INNER JOIN civicrm_contact c ON civicrm_relationship.contact_id_b = c.id
4349             $whereClause )
4350       ";
4351       CRM_Core_DAO::executeQuery($sql);
4352     }
4353   }
4354
4355   /**
4356    * Add relationship permission criteria to where clause.
4357    *
4358    * @param string $grouping
4359    * @param array $where Array to add "where" criteria to, in case you are generating a temp table.
4360    *   Not the main query.
4361    */
4362   public function addRelationshipPermissionClauses($grouping, &$where) {
4363     $relPermission = $this->getWhereValues('relation_permission', $grouping);
4364     if ($relPermission) {
4365       if (!is_array($relPermission[2])) {
4366         // this form value was scalar in previous versions of Civi
4367         $relPermission[2] = [$relPermission[2]];
4368       }
4369       $where[$grouping][] = "(civicrm_relationship.is_permission_a_b IN (" . implode(",", $relPermission[2]) . "))";
4370
4371       $allRelationshipPermissions = CRM_Contact_BAO_Relationship::buildOptions('is_permission_a_b');
4372
4373       $relPermNames = array_intersect_key($allRelationshipPermissions, array_flip($relPermission[2]));
4374       $this->_qill[$grouping][] = ts('Permissioned Relationships') . ' - ' . implode(' OR ', $relPermNames);
4375     }
4376   }
4377
4378   /**
4379    * Add start & end date criteria in
4380    * @param string $grouping
4381    * @param array $where
4382    *   = array to add where clauses to, in case you are generating a temp table.
4383    * not the main query.
4384    */
4385   public function addRelationshipDateClauses($grouping, &$where) {
4386     $dateValues = [];
4387     $dateTypes = [
4388       'start_date',
4389       'end_date',
4390     ];
4391
4392     foreach ($dateTypes as $dateField) {
4393       $dateValueLow = $this->getWhereValues('relation_' . $dateField . '_low', $grouping);
4394       $dateValueHigh = $this->getWhereValues('relation_' . $dateField . '_high', $grouping);
4395       if (!empty($dateValueLow)) {
4396         $date = date('Ymd', strtotime($dateValueLow[2]));
4397         $where[$grouping][] = "civicrm_relationship.$dateField >= $date";
4398         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or After') : ts('Relationship Recorded Start Date On or After')) . " " . CRM_Utils_Date::customFormat($date);
4399       }
4400       if (!empty($dateValueHigh)) {
4401         $date = date('Ymd', strtotime($dateValueHigh[2]));
4402         $where[$grouping][] = "civicrm_relationship.$dateField <= $date";
4403         $this->_qill[$grouping][] = ($dateField == 'end_date' ? ts('Relationship Ended on or Before') : ts('Relationship Recorded Start Date On or Before')) . " " . CRM_Utils_Date::customFormat($date);
4404       }
4405     }
4406   }
4407
4408   /**
4409    * Add start & end active period criteria in
4410    * @param string $grouping
4411    * @param array $where
4412    *   = array to add where clauses to, in case you are generating a temp table.
4413    * not the main query.
4414    */
4415   public function addRelationshipActivePeriodClauses($grouping, &$where) {
4416     $dateValues = [];
4417     $dateField = 'active_period_date';
4418
4419     $dateValueLow = $this->getWhereValues('relation_active_period_date_low', $grouping);
4420     $dateValueHigh = $this->getWhereValues('relation_active_period_date_high', $grouping);
4421     $dateValueLowFormated = $dateValueHighFormated = NULL;
4422     if (!empty($dateValueLow) && !empty($dateValueHigh)) {
4423       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4424       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4425       $this->_qill[$grouping][] = (ts('Relationship was active between')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated) . " and " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4426     }
4427     elseif (!empty($dateValueLow)) {
4428       $dateValueLowFormated = date('Ymd', strtotime($dateValueLow[2]));
4429       $this->_qill[$grouping][] = (ts('Relationship was active after')) . " " . CRM_Utils_Date::customFormat($dateValueLowFormated);
4430     }
4431     elseif (!empty($dateValueHigh)) {
4432       $dateValueHighFormated = date('Ymd', strtotime($dateValueHigh[2]));
4433       $this->_qill[$grouping][] = (ts('Relationship was active before')) . " " . CRM_Utils_Date::customFormat($dateValueHighFormated);
4434     }
4435
4436     if ($activePeriodClauses = self::getRelationshipActivePeriodClauses($dateValueLowFormated, $dateValueHighFormated, TRUE)) {
4437       $where[$grouping][] = $activePeriodClauses;
4438     }
4439   }
4440
4441   /**
4442    * Get start & end active period criteria
4443    */
4444   public static function getRelationshipActivePeriodClauses($from, $to, $forceTableName) {
4445     $tableName = $forceTableName ? 'civicrm_relationship.' : '';
4446     if (!is_null($from) && !is_null($to)) {
4447       return '(((' . $tableName . 'start_date >= ' . $from . ' AND ' . $tableName . 'start_date <= ' . $to . ') OR
4448                 (' . $tableName . 'end_date >= ' . $from . ' AND ' . $tableName . 'end_date <= ' . $to . ') OR
4449                 (' . $tableName . 'start_date <= ' . $from . ' AND ' . $tableName . 'end_date >= ' . $to . ' )) OR
4450                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4451                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . ') OR
4452                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4453     }
4454     elseif (!is_null($from)) {
4455       return '((' . $tableName . 'start_date >= ' . $from . ') OR
4456                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4457                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date >= ' . $from . '))';
4458     }
4459     elseif (!is_null($to)) {
4460       return '((' . $tableName . 'start_date <= ' . $to . ') OR
4461                (' . $tableName . 'start_date IS NULL AND ' . $tableName . 'end_date IS NULL) OR
4462                (' . $tableName . 'end_date IS NULL AND ' . $tableName . 'start_date <= ' . $to . '))';
4463     }
4464   }
4465
4466   /**
4467    * Default set of return properties.
4468    *
4469    * @param int $mode
4470    *
4471    * @return array
4472    *   derault return properties
4473    */
4474   public static function &defaultReturnProperties($mode = 1) {
4475     if (!isset(self::$_defaultReturnProperties)) {
4476       self::$_defaultReturnProperties = [];
4477     }
4478
4479     if (!isset(self::$_defaultReturnProperties[$mode])) {
4480       // add activity return properties
4481       if ($mode & CRM_Contact_BAO_Query::MODE_ACTIVITY) {
4482         self::$_defaultReturnProperties[$mode] = CRM_Activity_BAO_Query::defaultReturnProperties($mode, FALSE);
4483       }
4484       else {
4485         self::$_defaultReturnProperties[$mode] = CRM_Core_Component::defaultReturnProperties($mode, FALSE);
4486       }
4487
4488       if (empty(self::$_defaultReturnProperties[$mode])) {
4489         self::$_defaultReturnProperties[$mode] = [
4490           'home_URL' => 1,
4491           'image_URL' => 1,
4492           'legal_identifier' => 1,
4493           'external_identifier' => 1,
4494           'contact_type' => 1,
4495           'contact_sub_type' => 1,
4496           'sort_name' => 1,
4497           'display_name' => 1,
4498           'preferred_mail_format' => 1,
4499           'nick_name' => 1,
4500           'first_name' => 1,
4501           'middle_name' => 1,
4502           'last_name' => 1,
4503           'prefix_id' => 1,
4504           'suffix_id' => 1,
4505           'formal_title' => 1,
4506           'communication_style_id' => 1,
4507           'birth_date' => 1,
4508           'gender_id' => 1,
4509           'street_address' => 1,
4510           'supplemental_address_1' => 1,
4511           'supplemental_address_2' => 1,
4512           'supplemental_address_3' => 1,
4513           'city' => 1,
4514           'postal_code' => 1,
4515           'postal_code_suffix' => 1,
4516           'state_province' => 1,
4517           'country' => 1,
4518           'world_region' => 1,
4519           'geo_code_1' => 1,
4520           'geo_code_2' => 1,
4521           'email' => 1,
4522           'on_hold' => 1,
4523           'phone' => 1,
4524           'im' => 1,
4525           'household_name' => 1,
4526           'organization_name' => 1,
4527           'deceased_date' => 1,
4528           'is_deceased' => 1,
4529           'job_title' => 1,
4530           'legal_name' => 1,
4531           'sic_code' => 1,
4532           'current_employer' => 1,
4533           // FIXME: should we use defaultHierReturnProperties() for the below?
4534           'do_not_email' => 1,
4535           'do_not_mail' => 1,
4536           'do_not_sms' => 1,
4537           'do_not_phone' => 1,
4538           'do_not_trade' => 1,
4539           'is_opt_out' => 1,
4540           'contact_is_deleted' => 1,
4541           'preferred_communication_method' => 1,
4542           'preferred_language' => 1,
4543         ];
4544       }
4545     }
4546     return self::$_defaultReturnProperties[$mode];
4547   }
4548
4549   /**
4550    * Get primary condition for a sql clause.
4551    *
4552    * @param int $value
4553    *
4554    * @return string|NULL
4555    */
4556   public static function getPrimaryCondition($value) {
4557     if (is_numeric($value)) {
4558       $value = (int ) $value;
4559       return ($value == 1) ? 'is_primary = 1' : 'is_primary = 0';
4560     }
4561     return NULL;
4562   }
4563
4564   /**
4565    * Wrapper for a simple search query.
4566    *
4567    * @param array $params
4568    * @param array $returnProperties
4569    * @param bool $count
4570    *
4571    * @return string
4572    */
4573   public static function getQuery($params = NULL, $returnProperties = NULL, $count = FALSE) {
4574     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
4575     list($select, $from, $where, $having) = $query->query();
4576     $groupBy = ($query->_useGroupBy) ? 'GROUP BY contact_a.id' : '';
4577
4578     $query = "$select $from $where $groupBy $having";
4579     return $query;
4580   }
4581
4582   /**
4583    * These are stub comments as this function needs more explanation - particularly in terms of how it
4584    * relates to $this->searchQuery and why it replicates rather than calles $this->searchQuery.
4585    *
4586    * This function was originally written as a wrapper for the api query but is called from multiple places
4587    * in the core code directly so the name is misleading. This function does not use the searchQuery function
4588    * but it is unclear as to whehter that is historical or there is a reason
4589    *  CRM-11290 led to the permissioning action being extracted from searchQuery & shared with this function
4590    *
4591    * @param array $params
4592    * @param array $returnProperties
4593    * @param null $fields
4594    * @param string $sort
4595    * @param int $offset
4596    * @param int $row_count
4597    * @param bool $smartGroupCache
4598    *   ?? update smart group cache?.
4599    * @param bool $count
4600    *   Return count obnly.
4601    * @param bool $skipPermissions
4602    *   Should permissions be ignored or should the logged in user's permissions be applied.
4603    * @param int $mode
4604    *   This basically correlates to the component.
4605    * @param string $apiEntity
4606    *   The api entity being called.
4607    *   This sort-of duplicates $mode in a confusing way. Probably not by design.
4608    *
4609    * @param bool|null $primaryLocationOnly
4610    * @return array
4611    */
4612   public static function apiQuery(
4613     $params = NULL,
4614     $returnProperties = NULL,
4615     $fields = NULL,
4616     $sort = NULL,
4617     $offset = 0,
4618     $row_count = 25,
4619     $smartGroupCache = TRUE,
4620     $count = FALSE,
4621     $skipPermissions = TRUE,
4622     $mode = CRM_Contact_BAO_Query::MODE_CONTACTS,
4623     $apiEntity = NULL,
4624     $primaryLocationOnly = NULL
4625   ) {
4626
4627     $query = new CRM_Contact_BAO_Query(
4628       $params, $returnProperties,
4629       NULL, TRUE, FALSE, $mode,
4630       $skipPermissions,
4631       TRUE, $smartGroupCache,
4632       NULL, 'AND',
4633       $apiEntity, $primaryLocationOnly
4634     );
4635
4636     //this should add a check for view deleted if permissions are enabled
4637     if ($skipPermissions) {
4638       $query->_skipDeleteClause = TRUE;
4639     }
4640     $query->generatePermissionClause(FALSE, $count);
4641
4642     // note : this modifies _fromClause and _simpleFromClause
4643     $query->includePseudoFieldsJoin($sort);
4644
4645     list($select, $from, $where, $having) = $query->query($count);
4646
4647     $options = $query->_options;
4648     if (!empty($query->_permissionWhereClause)) {
4649       if (empty($where)) {
4650         $where = "WHERE $query->_permissionWhereClause";
4651       }
4652       else {
4653         $where = "$where AND $query->_permissionWhereClause";
4654       }
4655     }
4656
4657     $sql = "$select $from $where $having";
4658
4659     // add group by only when API action is not getcount
4660     //  otherwise query fetches incorrect count
4661     if ($query->_useGroupBy && !$count) {
4662       $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
4663     }
4664     if (!empty($sort)) {
4665       $sort = CRM_Utils_Type::escape($sort, 'String');
4666       $sql .= " ORDER BY $sort ";
4667     }
4668     if ($row_count > 0 && $offset >= 0) {
4669       $offset = CRM_Utils_Type::escape($offset, 'Int');
4670       $row_count = CRM_Utils_Type::escape($row_count, 'Int');
4671       $sql .= " LIMIT $offset, $row_count ";
4672     }
4673
4674     $dao = CRM_Core_DAO::executeQuery($sql);
4675
4676     // @todo derive this from the component class rather than hard-code two options.
4677     $entityIDField = ($mode == CRM_Contact_BAO_Query::MODE_CONTRIBUTE) ? 'contribution_id' : 'contact_id';
4678
4679     $values = [];
4680     while ($dao->fetch()) {
4681       if ($count) {
4682         $noRows = $dao->rowCount;
4683         return [$noRows, NULL];
4684       }
4685       $val = $query->store($dao);
4686       $convertedVals = $query->convertToPseudoNames($dao, TRUE, TRUE);
4687
4688       if (!empty($convertedVals)) {
4689         $val = array_replace_recursive($val, $convertedVals);
4690       }
4691       $values[$dao->$entityIDField] = $val;
4692     }
4693     return [$values, $options];
4694   }
4695
4696   /**
4697    * Get the actual custom field name by stripping off the appended string.
4698    *
4699    * The string could be _relative, _from, or _to
4700    *
4701    * @todo use metadata rather than convention to do this.
4702    *
4703    * @param string $parameterName
4704    *   The name of the parameter submitted to the form.
4705    *   e.g
4706    *   custom_3_relative
4707    *   custom_3_from
4708    *
4709    * @return string
4710    */
4711   public static function getCustomFieldName($parameterName) {
4712     if (substr($parameterName, -5, 5) == '_from') {
4713       return substr($parameterName, 0, strpos($parameterName, '_from'));
4714     }
4715     if (substr($parameterName, -9, 9) == '_relative') {
4716       return substr($parameterName, 0, strpos($parameterName, '_relative'));
4717     }
4718     if (substr($parameterName, -3, 3) == '_to') {
4719       return substr($parameterName, 0, strpos($parameterName, '_to'));
4720     }
4721   }
4722
4723   /**
4724    * Convert submitted values for relative custom fields to query object format.
4725    *
4726    * The query will support the sqlOperator format so convert to that format.
4727    *
4728    * @param array $formValues
4729    *   Submitted values.
4730    * @param array $params
4731    *   Converted parameters for the query object.
4732    * @param string $values
4733    *   Submitted value.
4734    * @param string $fieldName
4735    *   Submitted field name. (Matches form field not DB field.)
4736    */
4737   protected static function convertCustomRelativeFields(&$formValues, &$params, $values, $fieldName) {
4738     if (empty($values)) {
4739       // e.g we might have relative set & from & to empty. The form flow is a bit funky &
4740       // this function gets called again after they fields have been converted which can get ugly.
4741       return;
4742     }
4743     $customFieldName = self::getCustomFieldName($fieldName);
4744
4745     if (substr($fieldName, -9, 9) == '_relative') {
4746       list($from, $to) = CRM_Utils_Date::getFromTo($values, NULL, NULL);
4747     }
4748     else {
4749       if ($fieldName == $customFieldName . '_to' && CRM_Utils_Array::value($customFieldName . '_from', $formValues)) {
4750         // Both to & from are set. We only need to acton one, choosing from.
4751         return;
4752       }
4753
4754       $from = CRM_Utils_Array::value($customFieldName . '_from', $formValues, NULL);
4755       $to = CRM_Utils_Array::value($customFieldName . '_to', $formValues, NULL);
4756
4757       if (self::isCustomDateField($customFieldName)) {
4758         list($from, $to) = CRM_Utils_Date::getFromTo(NULL, $from, $to);
4759       }
4760     }
4761
4762     if ($from) {
4763       if ($to) {
4764         $relativeFunction = ['BETWEEN' => [$from, $to]];
4765       }
4766       else {
4767         $relativeFunction = ['>=' => $from];
4768       }
4769     }
4770     else {
4771       $relativeFunction = ['<=' => $to];
4772     }
4773     $params[] = [
4774       $customFieldName,
4775       '=',
4776       $relativeFunction,
4777       0,
4778       0,
4779     ];
4780   }
4781
4782   /**
4783    * Are we dealing with custom field of type date.
4784    *
4785    * @param $fieldName
4786    *
4787    * @return bool
4788    */
4789   public static function isCustomDateField($fieldName) {
4790     if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($fieldName)) == FALSE) {
4791       return FALSE;
4792     }
4793     if ('Date' == civicrm_api3('CustomField', 'getvalue', ['id' => $customFieldID, 'return' => 'data_type'])) {
4794       return TRUE;
4795     }
4796     return FALSE;
4797   }
4798
4799   /**
4800    * Has this field already been reformatting to Query object syntax.
4801    *
4802    * The form layer passed formValues to this function in preProcess & postProcess. Reason unknown. This seems
4803    * to come with associated double queries & is possibly damaging performance.
4804    *
4805    * However, here we add a tested function to ensure convertFormValues identifies pre-processed fields & returns
4806    * them as they are.
4807    *
4808    * @param mixed $values
4809    *   Value in formValues for the field.
4810    *
4811    * @return bool;
4812    */
4813   public static function isAlreadyProcessedForQueryFormat($values) {
4814     if (!is_array($values)) {
4815       return FALSE;
4816     }
4817     if (($operator = CRM_Utils_Array::value(1, $values)) == FALSE) {
4818       return FALSE;
4819     }
4820     return in_array($operator, CRM_Core_DAO::acceptedSQLOperators());
4821   }
4822
4823   /**
4824    * If the state and country are passed remove state.
4825    *
4826    * Country is implicit from the state, but including both results in
4827    * a poor query as there is no combined index on state AND country.
4828    *
4829    * CRM-18125
4830    *
4831    * @param array $formValues
4832    */
4833   public static function filterCountryFromValuesIfStateExists(&$formValues) {
4834     if (!empty($formValues['country']) && !empty($formValues['state_province'])) {
4835       // The use of array map sanitises the data by ensuring we are dealing with integers.
4836       $states = implode(', ', array_map('intval', $formValues['state_province']));
4837       $countryList = CRM_Core_DAO::singleValueQuery(
4838         "SELECT GROUP_CONCAT(country_id) FROM civicrm_state_province WHERE id IN ($states)"
4839       );
4840       if ($countryList == $formValues['country']) {
4841         unset($formValues['country']);
4842       }
4843     }
4844   }
4845
4846   /**
4847    * For some special cases, grouping by subset of select fields becomes mandatory.
4848    * Hence, full_group_by mode is handled by appending any_value
4849    * keyword to select fields not present in groupBy
4850    *
4851    * @param array $selectClauses
4852    * @param array $groupBy - Columns already included in GROUP By clause.
4853    * @param string $aggregateFunction
4854    *
4855    * @return string
4856    */
4857   public static function appendAnyValueToSelect($selectClauses, $groupBy, $aggregateFunction = 'ANY_VALUE') {
4858     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4859       $groupBy = array_map('trim', (array) $groupBy);
4860       $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4861       foreach ($selectClauses as $key => &$val) {
4862         list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
4863         // append ANY_VALUE() keyword
4864         if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
4865           $val = ($aggregateFunction == 'GROUP_CONCAT') ?
4866             str_replace($selectColumn, "$aggregateFunction(DISTINCT {$selectColumn})", $val) :
4867             str_replace($selectColumn, "$aggregateFunction({$selectColumn})", $val);
4868         }
4869       }
4870     }
4871
4872     return "SELECT " . implode(', ', $selectClauses) . " ";
4873   }
4874
4875   /**
4876    * For some special cases, where if non-aggregate ORDER BY columns are not present in GROUP BY
4877    *  on full_group_by mode, then append the those missing columns to GROUP BY clause
4878    * keyword to select fields not present in groupBy
4879    *
4880    * @param string $groupBy - GROUP BY clause where missing ORDER BY columns will be appended if not present
4881    * @param array $orderBys - ORDER BY sub-clauses
4882    *
4883    */
4884   public static function getGroupByFromOrderBy(&$groupBy, $orderBys) {
4885     if (!CRM_Utils_SQL::disableFullGroupByMode()) {
4886       foreach ($orderBys as $orderBy) {
4887         // remove sort syntax from ORDER BY clauses if present
4888         $orderBy = str_ireplace([' DESC', ' ASC', '`'], '', $orderBy);
4889         // if ORDER BY column is not present in GROUP BY then append it to end
4890         if (preg_match('/(MAX|MIN)\(/i', trim($orderBy)) !== 1 && !strstr($groupBy, $orderBy)) {
4891           $groupBy .= ", {$orderBy}";
4892         }
4893       }
4894     }
4895   }
4896
4897   /**
4898    * Include Select columns in groupBy clause.
4899    *
4900    * @param array $selectClauses
4901    * @param array $groupBy - Columns already included in GROUP By clause.
4902    *
4903    * @return string
4904    */
4905   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
4906     $groupBy = (array) $groupBy;
4907     $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
4908     $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
4909
4910     //return if ONLY_FULL_GROUP_BY is not enabled.
4911     if (CRM_Utils_SQL::supportsFullGroupBy() && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
4912       $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN|IF)[[:blank:]]*\(/i';
4913       foreach ($selectClauses as $key => $val) {
4914         $aliasArray = preg_split('/ as /i', $val);
4915         // if more than 1 alias we need to split by ','.
4916         if (count($aliasArray) > 2) {
4917           $aliasArray = preg_split('/,/', $val);
4918           foreach ($aliasArray as $key => $value) {
4919             $alias = current(preg_split('/ as /i', $value));
4920             if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
4921               $groupBy[] = $alias;
4922             }
4923           }
4924         }
4925         else {
4926           list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
4927           $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
4928           $tableName = current(explode('.', $selectColumn));
4929           $primaryKey = "{$tableName}.id";
4930           // exclude columns which are already included in groupBy and aggregate functions from select
4931           // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
4932           if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
4933             if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
4934               $groupBy[] = $alias;
4935             }
4936             else {
4937               $groupBy[] = $selectColumn;
4938             }
4939           }
4940         }
4941       }
4942     }
4943
4944     if (!empty($groupBy)) {
4945       return " GROUP BY " . implode(', ', $groupBy);
4946     }
4947     return '';
4948   }
4949
4950   /**
4951    * Create and query the db for an contact search.
4952    *
4953    * @param int $offset
4954    *   The offset for the query.
4955    * @param int $rowCount
4956    *   The number of rows to return.
4957    * @param string|CRM_Utils_Sort $sort
4958    *   The order by string.
4959    * @param bool $count
4960    *   Is this a count only query ?.
4961    * @param bool $includeContactIds
4962    *   Should we include contact ids?.
4963    * @param bool $sortByChar
4964    *   If true returns the distinct array of first characters for search results.
4965    * @param bool $groupContacts
4966    *   If true, return only the contact ids.
4967    * @param bool $returnQuery
4968    *   Should we return the query as a string.
4969    * @param string $additionalWhereClause
4970    *   If the caller wants to further restrict the search (used for components).
4971    * @param null $sortOrder
4972    * @param string $additionalFromClause
4973    *   Should be clause with proper joins, effective to reduce where clause load.
4974    *
4975    * @param bool $skipOrderAndLimit
4976    *
4977    * @return CRM_Core_DAO
4978    */
4979   public function searchQuery(
4980     $offset = 0, $rowCount = 0, $sort = NULL,
4981     $count = FALSE, $includeContactIds = FALSE,
4982     $sortByChar = FALSE, $groupContacts = FALSE,
4983     $returnQuery = FALSE,
4984     $additionalWhereClause = NULL, $sortOrder = NULL,
4985     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE
4986   ) {
4987
4988     $query = $this->getSearchSQL($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause, $skipOrderAndLimit);
4989
4990     if ($returnQuery) {
4991       return $query;
4992     }
4993     if ($count) {
4994       return CRM_Core_DAO::singleValueQuery($query);
4995     }
4996
4997     $dao = CRM_Core_DAO::executeQuery($query);
4998
4999     // We can always call this - it will only re-enable if it was originally enabled.
5000     CRM_Core_DAO::reenableFullGroupByMode();
5001
5002     if ($groupContacts) {
5003       $ids = [];
5004       while ($dao->fetch()) {
5005         $ids[] = $dao->id;
5006       }
5007       return implode(',', $ids);
5008     }
5009
5010     return $dao;
5011   }
5012
5013   /**
5014    * Create and query the db for the list of all first letters used by contacts
5015    *
5016    * @return CRM_Core_DAO
5017    */
5018   public function alphabetQuery() {
5019     $sqlParts = $this->getSearchSQLParts(NULL, NULL, NULL, FALSE, FALSE, TRUE);
5020     $query = "SELECT DISTINCT LEFT(contact_a.sort_name, 1) as sort_name
5021       {$sqlParts['from']}
5022       {$sqlParts['where']}
5023       {$sqlParts['having']}
5024       GROUP BY sort_name
5025       ORDER BY sort_name asc";
5026     $dao = CRM_Core_DAO::executeQuery($query);
5027     return $dao;
5028   }
5029
5030   /**
5031    * Fetch a list of contacts for displaying a search results page
5032    *
5033    * @param array $cids
5034    *   List of contact IDs
5035    * @param bool $includeContactIds
5036    * @return CRM_Core_DAO
5037    */
5038   public function getCachedContacts($cids, $includeContactIds) {
5039     CRM_Core_DAO::disableFullGroupByMode();
5040     CRM_Utils_Type::validateAll($cids, 'Positive');
5041     $this->_includeContactIds = $includeContactIds;
5042     $onlyDeleted = in_array(['deleted_contacts', '=', '1', '0', '0'], $this->_params);
5043     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
5044     $select .= sprintf(", (%s) AS _wgt", $this->createSqlCase('contact_a.id', $cids));
5045     $where .= sprintf(' AND contact_a.id IN (%s)', implode(',', $cids));
5046     $order = 'ORDER BY _wgt';
5047     $groupBy = $this->_useGroupBy ? ' GROUP BY contact_a.id' : '';
5048     $limit = '';
5049     $query = "$select $from $where $groupBy $order $limit";
5050
5051     $result = CRM_Core_DAO::executeQuery($query);
5052     CRM_Core_DAO::reenableFullGroupByMode();
5053     return $result;
5054   }
5055
5056   /**
5057    * Construct a SQL CASE expression.
5058    *
5059    * @param string $idCol
5060    *   The name of a column with ID's (eg 'contact_a.id').
5061    * @param array $cids
5062    *   Array(int $weight => int $id).
5063    * @return string
5064    *   CASE WHEN id=123 THEN 1 WHEN id=456 THEN 2 END
5065    */
5066   private function createSqlCase($idCol, $cids) {
5067     $buf = "CASE\n";
5068     foreach ($cids as $weight => $cid) {
5069       $buf .= " WHEN $idCol = $cid THEN $weight \n";
5070     }
5071     $buf .= "END\n";
5072     return $buf;
5073   }
5074
5075   /**
5076    * Populate $this->_permissionWhereClause with permission related clause and update other
5077    * query related properties.
5078    *
5079    * Function calls ACL permission class and hooks to filter the query appropriately
5080    *
5081    * Note that these 2 params were in the code when extracted from another function
5082    * and a second round extraction would be to make them properties of the class
5083    *
5084    * @param bool $onlyDeleted
5085    *   Only get deleted contacts.
5086    * @param bool $count
5087    *   Return Count only.
5088    */
5089   public function generatePermissionClause($onlyDeleted = FALSE, $count = FALSE) {
5090     if (!$this->_skipPermission) {
5091       $this->_permissionWhereClause = CRM_ACL_API::whereClause(
5092         CRM_Core_Permission::VIEW,
5093         $this->_tables,
5094         $this->_whereTables,
5095         NULL,
5096         $onlyDeleted,
5097         $this->_skipDeleteClause
5098       );
5099
5100       // regenerate fromClause since permission might have added tables
5101       if ($this->_permissionWhereClause) {
5102         //fix for row count in qill (in contribute/membership find)
5103         if (!$count) {
5104           $this->_useDistinct = TRUE;
5105         }
5106         //CRM-15231
5107         $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
5108         $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
5109         // note : this modifies _fromClause and _simpleFromClause
5110         $this->includePseudoFieldsJoin($this->_sort);
5111       }
5112     }
5113     else {
5114       // add delete clause if needed even if we are skipping permission
5115       // CRM-7639
5116       if (!$this->_skipDeleteClause) {
5117         if (CRM_Core_Permission::check('access deleted contacts') and $onlyDeleted) {
5118           $this->_permissionWhereClause = '(contact_a.is_deleted)';
5119         }
5120         else {
5121           // CRM-6181
5122           $this->_permissionWhereClause = '(contact_a.is_deleted = 0)';
5123         }
5124       }
5125     }
5126   }
5127
5128   /**
5129    * @param $val
5130    */
5131   public function setSkipPermission($val) {
5132     $this->_skipPermission = $val;
5133   }
5134
5135   /**
5136    * @param null $context
5137    *
5138    * @return array
5139    */
5140   public function summaryContribution($context = NULL) {
5141     list($innerselect, $from, $where, $having) = $this->query(TRUE);
5142     if ($this->_permissionWhereClause) {
5143       $where .= " AND " . $this->_permissionWhereClause;
5144     }
5145     if ($context == 'search') {
5146       $where .= " AND contact_a.is_deleted = 0 ";
5147     }
5148
5149     $this->appendFinancialTypeWhereAndFromToQueryStrings($where, $from);
5150
5151     $summary = ['total' => []];
5152     $this->addBasicStatsToSummary($summary, $where, $from);
5153
5154     if (CRM_Contribute_BAO_Query::isSoftCreditOptionEnabled()) {
5155       $this->addBasicSoftCreditStatsToStats($summary, $where, $from);
5156     }
5157
5158     $this->addBasicCancelStatsToSummary($summary, $where, $from);
5159
5160     return $summary;
5161   }
5162
5163   /**
5164    * Append financial ACL limits to the query from & where clauses, if applicable.
5165    *
5166    * @param string $where
5167    * @param string $from
5168    */
5169   public function appendFinancialTypeWhereAndFromToQueryStrings(&$where, &$from) {
5170     if (!CRM_Financial_BAO_FinancialType::isACLFinancialTypeStatus()) {
5171       return;
5172     }
5173     CRM_Financial_BAO_FinancialType::getAvailableFinancialTypes($financialTypes);
5174     if (!empty($financialTypes)) {
5175       $where .= " AND civicrm_contribution.financial_type_id IN (" . implode(',', array_keys($financialTypes)) . ") AND li.id IS NULL";
5176       $from .= " LEFT JOIN civicrm_line_item li
5177                   ON civicrm_contribution.id = li.contribution_id AND
5178                      li.entity_table = 'civicrm_contribution' AND li.financial_type_id NOT IN (" . implode(',', array_keys($financialTypes)) . ") ";
5179     }
5180     else {
5181       $where .= " AND civicrm_contribution.financial_type_id IN (0)";
5182     }
5183   }
5184
5185   /**
5186    * Getter for the qill object.
5187    *
5188    * @return string
5189    */
5190   public function qill() {
5191     return $this->_qill;
5192   }
5193
5194   /**
5195    * Default set of return default hier return properties.
5196    *
5197    * @return array
5198    */
5199   public static function &defaultHierReturnProperties() {
5200     if (!isset(self::$_defaultHierReturnProperties)) {
5201       self::$_defaultHierReturnProperties = [
5202         'home_URL' => 1,
5203         'image_URL' => 1,
5204         'legal_identifier' => 1,
5205         'external_identifier' => 1,
5206         'contact_type' => 1,
5207         'contact_sub_type' => 1,
5208         'sort_name' => 1,
5209         'display_name' => 1,
5210         'nick_name' => 1,
5211         'first_name' => 1,
5212         'middle_name' => 1,
5213         'last_name' => 1,
5214         'prefix_id' => 1,
5215         'suffix_id' => 1,
5216         'formal_title' => 1,
5217         'communication_style_id' => 1,
5218         'email_greeting' => 1,
5219         'postal_greeting' => 1,
5220         'addressee' => 1,
5221         'birth_date' => 1,
5222         'gender_id' => 1,
5223         'preferred_communication_method' => 1,
5224         'do_not_phone' => 1,
5225         'do_not_email' => 1,
5226         'do_not_mail' => 1,
5227         'do_not_sms' => 1,
5228         'do_not_trade' => 1,
5229         'location' => [
5230           '1' => [
5231             'location_type' => 1,
5232             'street_address' => 1,
5233             'city' => 1,
5234             'state_province' => 1,
5235             'postal_code' => 1,
5236             'postal_code_suffix' => 1,
5237             'country' => 1,
5238             'phone-Phone' => 1,
5239             'phone-Mobile' => 1,
5240             'phone-Fax' => 1,
5241             'phone-1' => 1,
5242             'phone-2' => 1,
5243             'phone-3' => 1,
5244             'im-1' => 1,
5245             'im-2' => 1,
5246             'im-3' => 1,
5247             'email-1' => 1,
5248             'email-2' => 1,
5249             'email-3' => 1,
5250           ],
5251           '2' => [
5252             'location_type' => 1,
5253             'street_address' => 1,
5254             'city' => 1,
5255             'state_province' => 1,
5256             'postal_code' => 1,
5257             'postal_code_suffix' => 1,
5258             'country' => 1,
5259             'phone-Phone' => 1,
5260             'phone-Mobile' => 1,
5261             'phone-1' => 1,
5262             'phone-2' => 1,
5263             'phone-3' => 1,
5264             'im-1' => 1,
5265             'im-2' => 1,
5266             'im-3' => 1,
5267             'email-1' => 1,
5268             'email-2' => 1,
5269             'email-3' => 1,
5270           ],
5271         ],
5272       ];
5273     }
5274     return self::$_defaultHierReturnProperties;
5275   }
5276
5277   /**
5278    * Build query for a date field.
5279    *
5280    * @param array $values
5281    * @param string $tableName
5282    * @param string $fieldName
5283    * @param string $dbFieldName
5284    * @param string $fieldTitle
5285    * @param bool $appendTimeStamp
5286    * @param string $dateFormat
5287    */
5288   public function dateQueryBuilder(
5289     $values, $tableName, $fieldName,
5290     $dbFieldName, $fieldTitle,
5291     $appendTimeStamp = TRUE,
5292     $dateFormat = 'YmdHis'
5293   ) {
5294     // @todo - remove dateFormat - pretty sure it's never passed in...
5295     list($name, $op, $value, $grouping, $wildcard) = $values;
5296
5297     if ($name == "{$fieldName}_low" ||
5298       $name == "{$fieldName}_high"
5299     ) {
5300       if (isset($this->_rangeCache[$fieldName]) || !$value) {
5301         return;
5302       }
5303       $this->_rangeCache[$fieldName] = 1;
5304
5305       $secondOP = $secondPhrase = $secondValue = $secondDate = $secondDateFormat = NULL;
5306
5307       if ($name == $fieldName . '_low') {
5308         $firstOP = '>=';
5309         $firstPhrase = ts('greater than or equal to');
5310         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5311
5312         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5313         if (!empty($secondValues) && $secondValues[2]) {
5314           $secondOP = '<=';
5315           $secondPhrase = ts('less than or equal to');
5316           $secondValue = $secondValues[2];
5317
5318           if ($appendTimeStamp && strlen($secondValue) == 10) {
5319             $secondValue .= ' 23:59:59';
5320           }
5321           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5322         }
5323       }
5324       elseif ($name == $fieldName . '_high') {
5325         $firstOP = '<=';
5326         $firstPhrase = ts('less than or equal to');
5327
5328         if ($appendTimeStamp && strlen($value) == 10) {
5329           $value .= ' 23:59:59';
5330         }
5331         $firstDate = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5332
5333         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5334         if (!empty($secondValues) && $secondValues[2]) {
5335           $secondOP = '>=';
5336           $secondPhrase = ts('greater than or equal to');
5337           $secondValue = $secondValues[2];
5338           $secondDate = CRM_Utils_Date::processDate($secondValue, NULL, FALSE, $dateFormat);
5339         }
5340       }
5341
5342       if (!$appendTimeStamp) {
5343         $firstDate = substr($firstDate, 0, 8);
5344       }
5345       $firstDateFormat = CRM_Utils_Date::customFormat($firstDate);
5346
5347       if ($secondDate) {
5348         if (!$appendTimeStamp) {
5349           $secondDate = substr($secondDate, 0, 8);
5350         }
5351         $secondDateFormat = CRM_Utils_Date::customFormat($secondDate);
5352       }
5353
5354       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5355       if ($secondDate) {
5356         $this->_where[$grouping][] = "
5357 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5358 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5359 ";
5360         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\" " . ts('AND') . " $secondPhrase \"$secondDateFormat\"";
5361       }
5362       else {
5363         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5364         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$firstDateFormat\"";
5365       }
5366     }
5367
5368     if ($name == $fieldName) {
5369       //In Get API, for operators other then '=' the $value is in array(op => value) format
5370       if (is_array($value) && !empty($value) && in_array(key($value), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
5371         $op = key($value);
5372         $value = $value[$op];
5373       }
5374
5375       $date = $format = NULL;
5376       if (strstr($op, 'IN')) {
5377         $format = [];
5378         foreach ($value as &$date) {
5379           $date = CRM_Utils_Date::processDate($date, NULL, FALSE, $dateFormat);
5380           if (!$appendTimeStamp) {
5381             $date = substr($date, 0, 8);
5382           }
5383           $format[] = CRM_Utils_Date::customFormat($date);
5384         }
5385         $date = "('" . implode("','", $value) . "')";
5386         $format = implode(', ', $format);
5387       }
5388       elseif ($value && (!strstr($op, 'NULL') && !strstr($op, 'EMPTY'))) {
5389         $date = CRM_Utils_Date::processDate($value, NULL, FALSE, $dateFormat);
5390         if (!$appendTimeStamp) {
5391           $date = substr($date, 0, 8);
5392         }
5393         $format = CRM_Utils_Date::customFormat($date);
5394         $date = "'$date'";
5395       }
5396
5397       if ($date) {
5398         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op $date";
5399       }
5400       else {
5401         $this->_where[$grouping][] = self::buildClause("{$tableName}.{$dbFieldName}", $op);
5402       }
5403
5404       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5405
5406       $op = CRM_Utils_Array::value($op, CRM_Core_SelectValues::getSearchBuilderOperators(), $op);
5407       $this->_qill[$grouping][] = "$fieldTitle $op $format";
5408     }
5409   }
5410
5411   /**
5412    * @param $values
5413    * @param string $tableName
5414    * @param string $fieldName
5415    * @param string $dbFieldName
5416    * @param $fieldTitle
5417    * @param null $options
5418    */
5419   public function numberRangeBuilder(
5420     &$values,
5421     $tableName, $fieldName,
5422     $dbFieldName, $fieldTitle,
5423     $options = NULL
5424   ) {
5425     list($name, $op, $value, $grouping, $wildcard) = $values;
5426
5427     if ($name == "{$fieldName}_low" ||
5428       $name == "{$fieldName}_high"
5429     ) {
5430       if (isset($this->_rangeCache[$fieldName])) {
5431         return;
5432       }
5433       $this->_rangeCache[$fieldName] = 1;
5434
5435       $secondOP = $secondPhrase = $secondValue = NULL;
5436
5437       if ($name == "{$fieldName}_low") {
5438         $firstOP = '>=';
5439         $firstPhrase = ts('greater than');
5440
5441         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5442         if (!empty($secondValues)) {
5443           $secondOP = '<=';
5444           $secondPhrase = ts('less than');
5445           $secondValue = $secondValues[2];
5446         }
5447       }
5448       else {
5449         $firstOP = '<=';
5450         $firstPhrase = ts('less than');
5451
5452         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5453         if (!empty($secondValues)) {
5454           $secondOP = '>=';
5455           $secondPhrase = ts('greater than');
5456           $secondValue = $secondValues[2];
5457         }
5458       }
5459
5460       if ($secondOP) {
5461         $this->_where[$grouping][] = "
5462 ( {$tableName}.{$dbFieldName} $firstOP {$value} ) AND
5463 ( {$tableName}.{$dbFieldName} $secondOP {$secondValue} )
5464 ";
5465         $displayValue = $options ? $options[$value] : $value;
5466         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5467
5468         $this->_qill[$grouping][]
5469           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5470       }
5471       else {
5472         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP {$value}";
5473         $displayValue = $options ? $options[$value] : $value;
5474         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5475       }
5476       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5477
5478       return;
5479     }
5480
5481     if ($name == $fieldName) {
5482       $op = '=';
5483       $phrase = '=';
5484
5485       $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $op {$value}";
5486
5487       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5488       $displayValue = $options ? $options[$value] : $value;
5489       $this->_qill[$grouping][] = "$fieldTitle - $phrase \"$displayValue\"";
5490     }
5491   }
5492
5493   /**
5494    * @param $values
5495    * @param string $tableName
5496    * @param string $fieldName
5497    * @param string $dbFieldName
5498    * @param $fieldTitle
5499    * @param null $options
5500    */
5501   public function ageRangeQueryBuilder(
5502     &$values,
5503     $tableName, $fieldName,
5504     $dbFieldName, $fieldTitle,
5505     $options = NULL
5506   ) {
5507     list($name, $op, $value, $grouping, $wildcard) = $values;
5508
5509     $asofDateValues = $this->getWhereValues("{$fieldName}_asof_date", $grouping);
5510     // will be treated as current day
5511     $asofDate = NULL;
5512     if ($asofDateValues) {
5513       $asofDate = CRM_Utils_Date::processDate($asofDateValues[2]);
5514       $asofDateFormat = CRM_Utils_Date::customFormat(substr($asofDate, 0, 8));
5515       $fieldTitle .= ' ' . ts('as of') . ' ' . $asofDateFormat;
5516     }
5517
5518     if ($name == "{$fieldName}_low" ||
5519       $name == "{$fieldName}_high"
5520     ) {
5521       if (isset($this->_rangeCache[$fieldName])) {
5522         return;
5523       }
5524       $this->_rangeCache[$fieldName] = 1;
5525
5526       $secondOP = $secondPhrase = $secondValue = NULL;
5527
5528       if ($name == "{$fieldName}_low") {
5529         $firstPhrase = ts('greater than or equal to');
5530         // NB: age > X means date of birth < Y
5531         $firstOP = '<=';
5532         $firstDate = self::calcDateFromAge($asofDate, $value, 'min');
5533
5534         $secondValues = $this->getWhereValues("{$fieldName}_high", $grouping);
5535         if (!empty($secondValues)) {
5536           $secondOP = '>=';
5537           $secondPhrase = ts('less than or equal to');
5538           $secondValue = $secondValues[2];
5539           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'max');
5540         }
5541       }
5542       else {
5543         $firstOP = '>=';
5544         $firstPhrase = ts('less than or equal to');
5545         $firstDate = self::calcDateFromAge($asofDate, $value, 'max');
5546
5547         $secondValues = $this->getWhereValues("{$fieldName}_low", $grouping);
5548         if (!empty($secondValues)) {
5549           $secondOP = '<=';
5550           $secondPhrase = ts('greater than or equal to');
5551           $secondValue = $secondValues[2];
5552           $secondDate = self::calcDateFromAge($asofDate, $secondValue, 'min');
5553         }
5554       }
5555
5556       if ($secondOP) {
5557         $this->_where[$grouping][] = "
5558 ( {$tableName}.{$dbFieldName} $firstOP '$firstDate' ) AND
5559 ( {$tableName}.{$dbFieldName} $secondOP '$secondDate' )
5560 ";
5561         $displayValue = $options ? $options[$value] : $value;
5562         $secondDisplayValue = $options ? $options[$secondValue] : $secondValue;
5563
5564         $this->_qill[$grouping][]
5565           = "$fieldTitle - $firstPhrase \"$displayValue\" " . ts('AND') . " $secondPhrase \"$secondDisplayValue\"";
5566       }
5567       else {
5568         $this->_where[$grouping][] = "{$tableName}.{$dbFieldName} $firstOP '$firstDate'";
5569         $displayValue = $options ? $options[$value] : $value;
5570         $this->_qill[$grouping][] = "$fieldTitle - $firstPhrase \"$displayValue\"";
5571       }
5572       $this->_tables[$tableName] = $this->_whereTables[$tableName] = 1;
5573       return;
5574     }
5575   }
5576
5577   /**
5578    * Calculate date from age.
5579    *
5580    * @param string $asofDate
5581    * @param int $age
5582    * @param string $type
5583    *
5584    * @return string
5585    */
5586   public static function calcDateFromAge($asofDate, $age, $type) {
5587     $date = new DateTime($asofDate);
5588     if ($type == "min") {
5589       // minimum age is $age: dob <= date - age "235959"
5590       $date->sub(new DateInterval("P" . $age . "Y"));
5591       return $date->format('Ymd') . "235959";
5592     }
5593     else {
5594       // max age is $age: dob >= date - (age + 1y) + 1d "000000"
5595       $date->sub(new DateInterval("P" . ($age + 1) . "Y"))->add(new DateInterval("P1D"));
5596       return $date->format('Ymd') . "000000";
5597     }
5598   }
5599
5600   /**
5601    * Given the field name, operator, value & its data type
5602    * builds the where Clause for the query
5603    * used for handling 'IS NULL'/'IS NOT NULL' operators
5604    *
5605    * @param string $field
5606    *   Fieldname.
5607    * @param string $op
5608    *   Operator.
5609    * @param string $value
5610    *   Value.
5611    * @param string $dataType
5612    *   Data type of the field.
5613    *
5614    * @return string
5615    *   Where clause for the query.
5616    */
5617   public static function buildClause($field, $op, $value = NULL, $dataType = NULL) {
5618     $op = trim($op);
5619     $clause = "$field $op";
5620
5621     switch ($op) {
5622       case 'IS NULL':
5623       case 'IS NOT NULL':
5624         return $clause;
5625
5626       case 'IS EMPTY':
5627         $clause = ($dataType == 'Date') ? " $field IS NULL " : " (NULLIF($field, '') IS NULL) ";
5628         return $clause;
5629
5630       case 'IS NOT EMPTY':
5631         $clause = ($dataType == 'Date') ? " $field IS NOT NULL " : " (NULLIF($field, '') IS NOT NULL) ";
5632         return $clause;
5633
5634       case 'RLIKE':
5635         return " {$clause} BINARY '{$value}' ";
5636
5637       case 'IN':
5638       case 'NOT IN':
5639         // I feel like this would be escaped properly if passed through $queryString = CRM_Core_DAO::createSqlFilter.
5640         if (!empty($value) && (!is_array($value) || !array_key_exists($op, $value))) {
5641           $value = [$op => (array) $value];
5642         }
5643
5644       default:
5645         if (empty($dataType) || $dataType == 'Date') {
5646           $dataType = 'String';
5647         }
5648         if (is_array($value)) {
5649           //this could have come from the api - as in the restWhere section we potentially use the api operator syntax which is becoming more
5650           // widely used and consistent across the codebase
5651           // adding this here won't accept the search functions which don't submit an array
5652           if (($queryString = CRM_Core_DAO::createSqlFilter($field, $value, $dataType)) != FALSE) {
5653
5654             return $queryString;
5655           }
5656           if (!empty($value[0]) && $op === 'BETWEEN') {
5657             CRM_Core_Error::deprecatedFunctionWarning('Fix search input params');
5658             if (($queryString = CRM_Core_DAO::createSqlFilter($field, [$op => $value], $dataType)) != FALSE) {
5659               return $queryString;
5660             }
5661           }
5662           throw new CRM_Core_Exception(ts('Failed to interpret input for search'));
5663         }
5664
5665         $value = CRM_Utils_Type::escape($value, $dataType);
5666         // if we don't have a dataType we should assume
5667         if ($dataType == 'String' || $dataType == 'Text') {
5668           $value = "'" . $value . "'";
5669         }
5670         return "$clause $value";
5671     }
5672   }
5673
5674   /**
5675    * @param bool $reset
5676    *
5677    * @return array
5678    */
5679   public function openedSearchPanes($reset = FALSE) {
5680     if (!$reset || empty($this->_whereTables)) {
5681       return self::$_openedPanes;
5682     }
5683
5684     // pane name to table mapper
5685     $panesMapper = [
5686       ts('Contributions') => 'civicrm_contribution',
5687       ts('Memberships') => 'civicrm_membership',
5688       ts('Events') => 'civicrm_participant',
5689       ts('Relationships') => 'civicrm_relationship',
5690       ts('Activities') => 'civicrm_activity',
5691       ts('Pledges') => 'civicrm_pledge',
5692       ts('Cases') => 'civicrm_case',
5693       ts('Grants') => 'civicrm_grant',
5694       ts('Address Fields') => 'civicrm_address',
5695       ts('Notes') => 'civicrm_note',
5696       ts('Change Log') => 'civicrm_log',
5697       ts('Mailings') => 'civicrm_mailing',
5698     ];
5699     CRM_Contact_BAO_Query_Hook::singleton()->getPanesMapper($panesMapper);
5700
5701     foreach (array_keys($this->_whereTables) as $table) {
5702       if ($panName = array_search($table, $panesMapper)) {
5703         self::$_openedPanes[$panName] = TRUE;
5704       }
5705     }
5706
5707     return self::$_openedPanes;
5708   }
5709
5710   /**
5711    * @param $operator
5712    */
5713   public function setOperator($operator) {
5714     $validOperators = ['AND', 'OR'];
5715     if (!in_array($operator, $validOperators)) {
5716       $operator = 'AND';
5717     }
5718     $this->_operator = $operator;
5719   }
5720
5721   /**
5722    * @return string
5723    */
5724   public function getOperator() {
5725     return $this->_operator;
5726   }
5727
5728   /**
5729    * @param $from
5730    * @param $where
5731    * @param $having
5732    */
5733   public function filterRelatedContacts(&$from, &$where, &$having) {
5734     if (!isset(Civi::$statics[__CLASS__]['related_contacts_filter'])) {
5735       Civi::$statics[__CLASS__]['related_contacts_filter'] = [];
5736     }
5737     $_rTempCache =& Civi::$statics[__CLASS__]['related_contacts_filter'];
5738     // since there only can be one instance of this filter in every query
5739     // skip if filter has already applied
5740     foreach ($_rTempCache as $acache) {
5741       foreach ($acache['queries'] as $aqcache) {
5742         if (strpos($from, $aqcache['from']) !== FALSE) {
5743           $having = NULL;
5744           return;
5745         }
5746       }
5747     }
5748     $arg_sig = sha1("$from $where $having");
5749     if (isset($_rTempCache[$arg_sig])) {
5750       $cache = $_rTempCache[$arg_sig];
5751     }
5752     else {
5753       // create temp table with contact ids
5754
5755       $tableName = CRM_Utils_SQL_TempTable::build()->createWithColumns('contact_id int primary key')->setMemory(TRUE)->getName();
5756
5757       $sql = "
5758 REPLACE INTO $tableName ( contact_id )
5759 SELECT contact_a.id
5760        $from
5761        $where
5762        $having
5763 ";
5764       CRM_Core_DAO::executeQuery($sql);
5765
5766       $cache = ['tableName' => $tableName, 'queries' => []];
5767       $_rTempCache[$arg_sig] = $cache;
5768     }
5769     // upsert the query depending on relationship type
5770     if (isset($cache['queries'][$this->_displayRelationshipType])) {
5771       $qcache = $cache['queries'][$this->_displayRelationshipType];
5772     }
5773     else {
5774       $tableName = $cache['tableName'];
5775       $qcache = [
5776         "from" => "",
5777         "where" => "",
5778       ];
5779       $rTypes = CRM_Core_PseudoConstant::relationshipType();
5780       if (is_numeric($this->_displayRelationshipType)) {
5781         $relationshipTypeLabel = $rTypes[$this->_displayRelationshipType]['label_a_b'];
5782         $qcache['from'] = "
5783 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id OR displayRelType.contact_id_b = contact_a.id )
5784 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a OR transform_temp.contact_id = displayRelType.contact_id_b )
5785 ";
5786         $qcache['where'] = "
5787 WHERE displayRelType.relationship_type_id = {$this->_displayRelationshipType}
5788 AND   displayRelType.is_active = 1
5789 ";
5790       }
5791       else {
5792         list($relType, $dirOne, $dirTwo) = explode('_', $this->_displayRelationshipType);
5793         if ($dirOne == 'a') {
5794           $relationshipTypeLabel = $rTypes[$relType]['label_a_b'];
5795           $qcache['from'] .= "
5796 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_a = contact_a.id )
5797 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_b )
5798 ";
5799         }
5800         else {
5801           $relationshipTypeLabel = $rTypes[$relType]['label_b_a'];
5802           $qcache['from'] .= "
5803 INNER JOIN civicrm_relationship displayRelType ON ( displayRelType.contact_id_b = contact_a.id )
5804 INNER JOIN $tableName transform_temp ON ( transform_temp.contact_id = displayRelType.contact_id_a )
5805 ";
5806         }
5807         $qcache['where'] = "
5808 WHERE displayRelType.relationship_type_id = $relType
5809 AND   displayRelType.is_active = 1
5810 ";
5811       }
5812       $qcache['relTypeLabel'] = $relationshipTypeLabel;
5813       $_rTempCache[$arg_sig]['queries'][$this->_displayRelationshipType] = $qcache;
5814     }
5815     $qillMessage = ts('Contacts with a Relationship Type of: ');
5816     $iqill = $qillMessage . "'" . $qcache['relTypeLabel'] . "'";
5817     if (!is_array($this->_qill[0]) || !in_array($iqill, $this->_qill[0])) {
5818       $this->_qill[0][] = $iqill;
5819     }
5820     if (strpos($from, $qcache['from']) === FALSE) {
5821       // lets replace all the INNER JOIN's in the $from so we dont exclude other data
5822       // this happens when we have an event_type in the quert (CRM-7969)
5823       $from = str_replace("INNER JOIN", "LEFT JOIN", $from);
5824       $from .= $qcache['from'];
5825       $where = $qcache['where'];
5826       if (!empty($this->_permissionWhereClause)) {
5827         $where .= "AND $this->_permissionWhereClause";
5828       }
5829     }
5830
5831     $having = NULL;
5832   }
5833
5834   /**
5835    * See CRM-19811 for why this is database hurty without apparent benefit.
5836    *
5837    * @param $op
5838    *
5839    * @return bool
5840    */
5841   public static function caseImportant($op) {
5842     return in_array($op, ['LIKE', 'IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY']) ? FALSE : TRUE;
5843   }
5844
5845   /**
5846    * @param $returnProperties
5847    * @param $prefix
5848    *
5849    * @return bool
5850    */
5851   public static function componentPresent(&$returnProperties, $prefix) {
5852     foreach ($returnProperties as $name => $dontCare) {
5853       if (substr($name, 0, strlen($prefix)) == $prefix) {
5854         return TRUE;
5855       }
5856     }
5857     return FALSE;
5858   }
5859
5860   /**
5861    * Builds the necessary structures for all fields that are similar to option value look-ups.
5862    *
5863    * @param string $name
5864    *   the name of the field.
5865    * @param string $op
5866    *   the sql operator, this function should handle ALL SQL operators.
5867    * @param string $value
5868    *   depends on the operator and who's calling the query builder.
5869    * @param int $grouping
5870    *   the index where to place the where clause.
5871    * @param string $daoName
5872    *   DAO Name.
5873    * @param array $field
5874    *   an array that contains various properties of the field identified by $name.
5875    * @param string $label
5876    *   The label for this field element.
5877    * @param string $dataType
5878    *   The data type for this element.
5879    * @param bool $useIDsOnly
5880    */
5881   public function optionValueQuery(
5882     $name,
5883     $op,
5884     $value,
5885     $grouping,
5886     $daoName = NULL,
5887     $field,
5888     $label,
5889     $dataType = 'String',
5890     $useIDsOnly = FALSE
5891   ) {
5892
5893     $pseudoFields = [
5894       'email_greeting',
5895       'postal_greeting',
5896       'addressee',
5897       'gender_id',
5898       'prefix_id',
5899       'suffix_id',
5900       'communication_style_id',
5901     ];
5902
5903     if ($useIDsOnly) {
5904       list($tableName, $fieldName) = explode('.', $field['where'], 2);
5905       if ($tableName == 'civicrm_contact') {
5906         $wc = "contact_a.$fieldName";
5907       }
5908       else {
5909         // Special handling for on_hold, so that we actually use the 'where'
5910         // property in order to limit the query by the on_hold status of the email,
5911         // instead of using email.id which would be nonsensical.
5912         if ($field['name'] == 'on_hold') {
5913           $wc = "{$field['where']}";
5914         }
5915         else {
5916           $wc = "$tableName.id";
5917         }
5918       }
5919     }
5920     else {
5921       CRM_Core_Error::deprecatedFunctionWarning('pass $ids to this method');
5922       $wc = "{$field['where']}";
5923     }
5924     if (in_array($name, $pseudoFields)) {
5925       if (!in_array($name, ['gender_id', 'prefix_id', 'suffix_id', 'communication_style_id'])) {
5926         $wc = "contact_a.{$name}_id";
5927       }
5928       $dataType = 'Positive';
5929       $value = (!$value) ? 0 : $value;
5930     }
5931     if ($name == "world_region") {
5932       $field['name'] = $name;
5933     }
5934
5935     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue($daoName, $field['name'], $value, $op);
5936     $this->_qill[$grouping][] = ts("%1 %2 %3", [1 => $label, 2 => $qillop, 3 => $qillVal]);
5937     $this->_where[$grouping][] = self::buildClause($wc, $op, $value, $dataType);
5938   }
5939
5940   /**
5941    * Check and explode a user defined numeric string into an array
5942    * this was the protocol used by search builder in the old old days before we had
5943    * super nice js widgets to do the hard work
5944    *
5945    * @param string $string
5946    * @param string $dataType
5947    *   The dataType we should check for the values, default integer.
5948    *
5949    * @return bool|array
5950    *   false if string does not match the pattern
5951    *   array of numeric values if string does match the pattern
5952    */
5953   public static function parseSearchBuilderString($string, $dataType = 'Integer') {
5954     $string = trim($string);
5955     if (substr($string, 0, 1) != '(' || substr($string, -1, 1) != ')') {
5956       return FALSE;
5957     }
5958
5959     $string = substr($string, 1, -1);
5960     $values = explode(',', $string);
5961     if (empty($values)) {
5962       return FALSE;
5963     }
5964
5965     $returnValues = [];
5966     foreach ($values as $v) {
5967       if ($dataType == 'Integer' && !is_numeric($v)) {
5968         return FALSE;
5969       }
5970       elseif ($dataType == 'String' && !is_string($v)) {
5971         return FALSE;
5972       }
5973       $returnValues[] = trim($v);
5974     }
5975
5976     if (empty($returnValues)) {
5977       return FALSE;
5978     }
5979
5980     return $returnValues;
5981   }
5982
5983   /**
5984    * Convert the pseudo constants id's to their names
5985    *
5986    * @param CRM_Core_DAO $dao
5987    * @param bool $return
5988    * @param bool $usedForAPI
5989    *
5990    * @return array|NULL
5991    */
5992   public function convertToPseudoNames(&$dao, $return = FALSE, $usedForAPI = FALSE) {
5993     if (empty($this->_pseudoConstantsSelect)) {
5994       return NULL;
5995     }
5996     $values = [];
5997     foreach ($this->_pseudoConstantsSelect as $key => $value) {
5998       if (!empty($this->_pseudoConstantsSelect[$key]['sorting'])) {
5999         continue;
6000       }
6001
6002       if (is_object($dao) && property_exists($dao, $value['idCol'])) {
6003         $val = $dao->{$value['idCol']};
6004         if ($key == 'groups') {
6005           $dao->groups = $this->convertGroupIDStringToLabelString($dao, $val);
6006           continue;
6007         }
6008
6009         if (CRM_Utils_System::isNull($val)) {
6010           $dao->$key = NULL;
6011         }
6012         elseif (!empty($value['pseudoconstant'])) {
6013           // If pseudoconstant is set that is kind of defacto for 'we have a bit more info about this'
6014           // and we can use the metadata to figure it out.
6015           // ideally this bit of IF will absorb & replace all the rest in time as we move to
6016           // more metadata based choices.
6017           if (strpos($val, CRM_Core_DAO::VALUE_SEPARATOR) !== FALSE) {
6018             $dbValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, trim($val, CRM_Core_DAO::VALUE_SEPARATOR));
6019             foreach ($dbValues as $pseudoValue) {
6020               $convertedValues[] = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $pseudoValue);
6021             }
6022
6023             $dao->$key = ($usedForAPI) ? $convertedValues : implode(', ', $convertedValues);
6024             $realFieldName = CRM_Utils_Array::value('field_name', $this->_pseudoConstantsSelect[$key]);
6025             if ($usedForAPI && $realFieldName) {
6026               // normally we would see 2 fields returned for pseudoConstants. An exception is
6027               // preferred_communication_method where there is no id-variant.
6028               // For the api we prioritise getting the real data returned.
6029               // over the resolved version
6030               $dao->$realFieldName = $dbValues;
6031             }
6032
6033           }
6034           else {
6035             // This is basically the same as the default but since we have the bao we can use
6036             // a cached function.
6037             $dao->$key = CRM_Core_PseudoConstant::getLabel($value['bao'], $value['idCol'], $val);
6038           }
6039         }
6040         elseif ($baoName = CRM_Utils_Array::value('bao', $value, NULL)) {
6041           //preserve id value
6042           $idColumn = "{$key}_id";
6043           $dao->$idColumn = $val;
6044
6045           if ($key == 'state_province_name') {
6046             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::stateProvince($val);
6047           }
6048           else {
6049             $dao->{$value['pseudoField']} = $dao->$key = CRM_Core_PseudoConstant::getLabel($baoName, $value['pseudoField'], $val);
6050           }
6051         }
6052         elseif ($value['pseudoField'] == 'state_province_abbreviation') {
6053           $dao->$key = CRM_Core_PseudoConstant::stateProvinceAbbreviation($val);
6054         }
6055         // @todo handle this in the section above for pseudoconstants.
6056         elseif (in_array($value['pseudoField'], ['participant_role_id', 'participant_role'])) {
6057           // @todo define bao on this & merge into the above condition.
6058           $viewValues = explode(CRM_Core_DAO::VALUE_SEPARATOR, $val);
6059
6060           if ($value['pseudoField'] == 'participant_role') {
6061             $pseudoOptions = CRM_Core_PseudoConstant::get('CRM_Event_DAO_Participant', 'role_id');
6062             foreach ($viewValues as $k => $v) {
6063               $viewValues[$k] = $pseudoOptions[$v];
6064             }
6065           }
6066           $dao->$key = ($usedForAPI && count($viewValues) > 1) ? $viewValues : implode(', ', $viewValues);
6067         }
6068         else {
6069           $labels = CRM_Core_OptionGroup::values($value['pseudoField']);
6070           $dao->$key = $labels[$val];
6071         }
6072
6073         // return converted values in array format
6074         if ($return) {
6075           if (strpos($key, '-') !== FALSE) {
6076             $keyVal = explode('-', $key);
6077             $current = &$values;
6078             $lastElement = array_pop($keyVal);
6079             foreach ($keyVal as $v) {
6080               if (!array_key_exists($v, $current)) {
6081                 $current[$v] = [];
6082               }
6083               $current = &$current[$v];
6084             }
6085             $current[$lastElement] = $dao->$key;
6086           }
6087           else {
6088             $values[$key] = $dao->$key;
6089           }
6090         }
6091       }
6092     }
6093     if (!$usedForAPI) {
6094       foreach ($this->legacyHackedFields as $realField => $labelField) {
6095         // This is a temporary routine for handling these fields while
6096         // we figure out how to handled them based on metadata in
6097         /// export and search builder. CRM-19815, CRM-19830.
6098         if (isset($dao->$realField) && is_numeric($dao->$realField) && isset($dao->$labelField)) {
6099           $dao->$realField = $dao->$labelField;
6100         }
6101       }
6102     }
6103     return $values;
6104   }
6105
6106   /**
6107    * Include pseudo fields LEFT JOIN.
6108    * @param string|array $sort can be a object or string
6109    *
6110    * @return array|NULL
6111    */
6112   public function includePseudoFieldsJoin($sort) {
6113     if (!$sort || empty($this->_pseudoConstantsSelect)) {
6114       return NULL;
6115     }
6116     $sort = is_string($sort) ? $sort : $sort->orderBy();
6117     $present = [];
6118
6119     foreach ($this->_pseudoConstantsSelect as $name => $value) {
6120       if (!empty($value['table'])) {
6121         $regex = "/({$value['table']}\.|{$name})/";
6122         if (preg_match($regex, $sort)) {
6123           $this->_elemnt[$value['element']] = 1;
6124           $this->_select[$value['element']] = $value['select'];
6125           $this->_pseudoConstantsSelect[$name]['sorting'] = 1;
6126           $present[$value['table']] = $value['join'];
6127         }
6128       }
6129     }
6130     $presentSimpleFrom = $present;
6131
6132     if (array_key_exists('civicrm_worldregion', $this->_whereTables) &&
6133       array_key_exists('civicrm_country', $presentSimpleFrom)
6134     ) {
6135       unset($presentSimpleFrom['civicrm_country']);
6136     }
6137     if (array_key_exists('civicrm_worldregion', $this->_tables) &&
6138       array_key_exists('civicrm_country', $present)
6139     ) {
6140       unset($present['civicrm_country']);
6141     }
6142
6143     $presentClause = $presentSimpleFromClause = NULL;
6144     if (!empty($present)) {
6145       $presentClause = implode(' ', $present);
6146     }
6147     if (!empty($presentSimpleFrom)) {
6148       $presentSimpleFromClause = implode(' ', $presentSimpleFrom);
6149     }
6150
6151     $this->_fromClause = $this->_fromClause . $presentClause;
6152     $this->_simpleFromClause = $this->_simpleFromClause . $presentSimpleFromClause;
6153
6154     return [$presentClause, $presentSimpleFromClause];
6155   }
6156
6157   /**
6158    * Build qill for field.
6159    *
6160    * Qill refers to the query detail visible on the UI.
6161    *
6162    * @param string $daoName
6163    * @param string $fieldName
6164    * @param mixed $fieldValue
6165    * @param string $op
6166    * @param array $pseudoExtraParam
6167    * @param int $type
6168    *   Type of the field per CRM_Utils_Type
6169    *
6170    * @return array
6171    */
6172   public static function buildQillForFieldValue(
6173     $daoName,
6174     $fieldName,
6175     $fieldValue,
6176     $op,
6177     $pseudoExtraParam = [],
6178     $type = CRM_Utils_Type::T_STRING
6179   ) {
6180     $qillOperators = CRM_Core_SelectValues::getSearchBuilderOperators();
6181
6182     //API usually have fieldValue format as array(operator => array(values)),
6183     //so we need to separate operator out of fieldValue param
6184     if (is_array($fieldValue) && in_array(key($fieldValue), CRM_Core_DAO::acceptedSQLOperators(), TRUE)) {
6185       $op = key($fieldValue);
6186       $fieldValue = $fieldValue[$op];
6187     }
6188
6189     // if Operator chosen is NULL/EMPTY then
6190     if (strpos($op, 'NULL') !== FALSE || strpos($op, 'EMPTY') !== FALSE) {
6191       return [CRM_Utils_Array::value($op, $qillOperators, $op), ''];
6192     }
6193
6194     if ($fieldName == 'activity_type_id') {
6195       $pseudoOptions = CRM_Core_PseudoConstant::activityType(TRUE, TRUE, FALSE, 'label', TRUE);
6196     }
6197     elseif ($fieldName == 'country_id') {
6198       $pseudoOptions = CRM_Core_PseudoConstant::country();
6199     }
6200     elseif ($fieldName == 'county_id') {
6201       $pseudoOptions = CRM_Core_PseudoConstant::county();
6202     }
6203     elseif ($fieldName == 'world_region') {
6204       $pseudoOptions = CRM_Core_PseudoConstant::worldRegion();
6205     }
6206     elseif ($daoName == 'CRM_Event_DAO_Event' && $fieldName == 'id') {
6207       $checkPermission = CRM_Utils_Array::value('check_permission', $pseudoExtraParam, TRUE);
6208       $pseudoOptions = CRM_Event_BAO_Event::getEvents(0, $fieldValue, TRUE, $checkPermission, TRUE);
6209     }
6210     elseif ($fieldName == 'contribution_product_id') {
6211       $pseudoOptions = CRM_Contribute_PseudoConstant::products();
6212     }
6213     elseif ($daoName == 'CRM_Contact_DAO_Group' && $fieldName == 'id') {
6214       $pseudoOptions = CRM_Core_PseudoConstant::group();
6215     }
6216     elseif ($daoName == 'CRM_Batch_BAO_EntityBatch' && $fieldName == 'batch_id') {
6217       $pseudoOptions = CRM_Contribute_PseudoConstant::batch();
6218     }
6219     elseif ($daoName) {
6220       $pseudoOptions = CRM_Core_PseudoConstant::get($daoName, $fieldName, $pseudoExtraParam);
6221     }
6222
6223     if (is_array($fieldValue)) {
6224       $qillString = [];
6225       if (!empty($pseudoOptions)) {
6226         foreach ((array) $fieldValue as $val) {
6227           $qillString[] = CRM_Utils_Array::value($val, $pseudoOptions, $val);
6228         }
6229         $fieldValue = implode(', ', $qillString);
6230       }
6231       else {
6232         if ($type == CRM_Utils_Type::T_DATE) {
6233           foreach ($fieldValue as $index => $value) {
6234             $fieldValue[$index] = CRM_Utils_Date::customFormat($value);
6235           }
6236         }
6237         $separator = ', ';
6238         // @todo - this is a bit specific (one operator).
6239         // However it is covered by a unit test so can be altered later with
6240         // some confidence.
6241         if ($op == 'BETWEEN') {
6242           $separator = ' AND ';
6243         }
6244         $fieldValue = implode($separator, $fieldValue);
6245       }
6246     }
6247     elseif (!empty($pseudoOptions) && array_key_exists($fieldValue, $pseudoOptions)) {
6248       $fieldValue = $pseudoOptions[$fieldValue];
6249     }
6250     elseif ($type === CRM_Utils_Type::T_DATE) {
6251       $fieldValue = CRM_Utils_Date::customFormat($fieldValue);
6252     }
6253
6254     return [CRM_Utils_Array::value($op, $qillOperators, $op), $fieldValue];
6255   }
6256
6257   /**
6258    * Alter value to reflect wildcard settings.
6259    *
6260    * The form will have tried to guess whether this is a good field to wildcard but there is
6261    * also a site-wide setting that specifies whether it is OK to append the wild card to the beginning
6262    * or only the end of the string
6263    *
6264    * @param bool $wildcard
6265    *   This is a bool made on an assessment 'elsewhere' on whether this is a good field to wildcard.
6266    * @param string $op
6267    *   Generally '=' or 'LIKE'.
6268    * @param string $value
6269    *   The search string.
6270    *
6271    * @return string
6272    */
6273   public static function getWildCardedValue($wildcard, $op, $value) {
6274     if ($wildcard && $op == 'LIKE') {
6275       if (CRM_Core_Config::singleton()->includeWildCardInName && (substr($value, 0, 1) != '%')) {
6276         return "%$value%";
6277       }
6278       else {
6279         return "$value%";
6280       }
6281     }
6282     else {
6283       return "$value";
6284     }
6285   }
6286
6287   /**
6288    * Process special fields of Search Form in OK (Operator in Key) format
6289    *
6290    * @param array $formValues
6291    * @param array $specialFields
6292    *    Special params to be processed
6293    * @param array $changeNames
6294    *   Array of fields whose name should be changed
6295    */
6296   public static function processSpecialFormValue(&$formValues, $specialFields, $changeNames = []) {
6297     // Array of special fields whose value are considered only for NULL or EMPTY operators
6298     $nullableFields = ['contribution_batch_id'];
6299
6300     foreach ($specialFields as $element) {
6301       $value = CRM_Utils_Array::value($element, $formValues);
6302       if ($value) {
6303         if (is_array($value)) {
6304           if (in_array($element, array_keys($changeNames))) {
6305             unset($formValues[$element]);
6306             $element = $changeNames[$element];
6307           }
6308           $formValues[$element] = ['IN' => $value];
6309         }
6310         elseif (in_array($value, ['IS NULL', 'IS NOT NULL', 'IS EMPTY', 'IS NOT EMPTY'])) {
6311           $formValues[$element] = [$value => 1];
6312         }
6313         elseif (!in_array($element, $nullableFields)) {
6314           // if wildcard is already present return searchString as it is OR append and/or prepend with wildcard
6315           $isWilcard = strstr($value, '%') ? FALSE : CRM_Core_Config::singleton()->includeWildCardInName;
6316           $formValues[$element] = ['LIKE' => self::getWildCardedValue($isWilcard, 'LIKE', $value)];
6317         }
6318       }
6319     }
6320   }
6321
6322   /**
6323    * Parse and assimilate the various sort options.
6324    *
6325    * Side-effect: if sorting on a common column from a related table (`city`, `postal_code`,
6326    * `email`), the related table may be joined automatically.
6327    *
6328    * At time of writing, this code is deeply flawed and should be rewritten. For the moment,
6329    * it's been extracted to a standalone function.
6330    *
6331    * @param string|CRM_Utils_Sort $sort
6332    *   The order by string.
6333    * @param null $sortOrder
6334    *   Who knows? Hu knows. He who knows Hu knows who.
6335    *
6336    * @return array
6337    *   list(string $orderByClause, string $additionalFromClause).
6338    */
6339   protected function prepareOrderBy($sort, $sortOrder) {
6340     $orderByArray = [];
6341     $orderBy = '';
6342
6343     if (CRM_Core_Config::singleton()->includeOrderByClause ||
6344       isset($this->_distinctComponentClause)
6345     ) {
6346       if ($sort) {
6347         if (is_string($sort)) {
6348           $orderBy = $sort;
6349         }
6350         else {
6351           $orderBy = trim($sort->orderBy());
6352         }
6353         // Deliberately remove the backticks again, as they mess up the evil
6354         // string munging below. This balanced by re-escaping before use.
6355         $orderBy = str_replace('`', '', $orderBy);
6356
6357         if (!empty($orderBy)) {
6358           // this is special case while searching for
6359           // change log CRM-1718
6360           if (preg_match('/sort_name/i', $orderBy)) {
6361             $orderBy = str_replace('sort_name', 'contact_a.sort_name', $orderBy);
6362           }
6363
6364           if ($sortOrder) {
6365             $orderBy .= " $sortOrder";
6366           }
6367
6368           // always add contact_a.id to the ORDER clause
6369           // so the order is deterministic
6370           if (strpos('contact_a.id', $orderBy) === FALSE) {
6371             $orderBy .= ", contact_a.id";
6372           }
6373         }
6374       }
6375       else {
6376         $orderBy = " contact_a.sort_name ASC, contact_a.id";
6377       }
6378     }
6379     if (!$orderBy) {
6380       return NULL;
6381     }
6382     // Remove this here & add it at the end for simplicity.
6383     $order = trim($orderBy);
6384     $orderByArray = explode(',', $order);
6385
6386     foreach ($orderByArray as $orderByClause) {
6387       $orderByClauseParts = explode(' ', trim($orderByClause));
6388       $field = $orderByClauseParts[0];
6389       $direction = isset($orderByClauseParts[1]) ? $orderByClauseParts[1] : 'asc';
6390       $fieldSpec = $this->getMetadataForRealField($field);
6391
6392       // This is a hacky add-in for primary address joins. Feel free to iterate as it is unit tested.
6393       // @todo much more cleanup on location handling in addHierarchical elements. Potentially
6394       // add keys to $this->fields to represent the actual keys for locations.
6395       if (empty($fieldSpec) && substr($field, 0, 2) === '1-') {
6396         $fieldSpec = $this->getMetadataForField(substr($field, 2));
6397         $this->addAddressTable('1-' . str_replace('civicrm_', '', $fieldSpec['table_name']), 'is_primary = 1');
6398       }
6399
6400       if ($this->_returnProperties === []) {
6401         if (!empty($fieldSpec['table_name']) && !isset($this->_tables[$fieldSpec['table_name']])) {
6402           $this->_tables[$fieldSpec['table_name']] = 1;
6403           $order = $fieldSpec['where'] . ' ' . $direction;
6404         }
6405
6406       }
6407       $cfID = CRM_Core_BAO_CustomField::getKeyID($field);
6408       // add to cfIDs array if not present
6409       if (!empty($cfID) && !array_key_exists($cfID, $this->_cfIDs)) {
6410         $this->_cfIDs[$cfID] = [];
6411         $this->_customQuery = new CRM_Core_BAO_CustomQuery($this->_cfIDs, TRUE, $this->_locationSpecificCustomFields);
6412         $this->_customQuery->query();
6413         $this->_select = array_merge($this->_select, $this->_customQuery->_select);
6414         $this->_tables = array_merge($this->_tables, $this->_customQuery->_tables);
6415       }
6416
6417       // By replacing the join to the option value table with the mysql construct
6418       // ORDER BY field('contribution_status_id', 2,1,4)
6419       // we can remove a join. In the case of the option value join it is
6420       /// a join known to cause slow queries.
6421       // @todo cover other pseudoconstant types. Limited to option group ones  & Foreign keys
6422       // matching an id+name parrern in the
6423       // first instance for scope reasons. They require slightly different handling as the column (label)
6424       // is not declared for them.
6425       // @todo so far only integer fields are being handled. If we add string fields we need to look at
6426       // escaping.
6427       $pseudoConstantMetadata = CRM_Utils_Array::value('pseudoconstant', $fieldSpec, FALSE);
6428       if (!empty($pseudoConstantMetadata)
6429       ) {
6430         if (!empty($pseudoConstantMetadata['optionGroupName'])
6431           || $this->isPseudoFieldAnFK($fieldSpec)
6432         ) {
6433           // dev/core#1305 @todo this is not the right thing to do but for now avoid fatal error
6434           if (empty($fieldSpec['bao'])) {
6435             continue;
6436           }
6437           $sortedOptions = $fieldSpec['bao']::buildOptions($fieldSpec['name'], NULL, [
6438             'orderColumn' => CRM_Utils_Array::value('labelColumn', $pseudoConstantMetadata, 'label'),
6439           ]);
6440           $fieldIDsInOrder = implode(',', array_keys($sortedOptions));
6441           // Pretty sure this validation ALSO happens in the order clause & this can't be reached but...
6442           // this might give some early warning.
6443           CRM_Utils_Type::validate($fieldIDsInOrder, 'CommaSeparatedIntegers');
6444           $order = str_replace("$field", "field({$fieldSpec['name']},$fieldIDsInOrder)", $order);
6445         }
6446         //CRM-12565 add "`" around $field if it is a pseudo constant
6447         // This appears to be for 'special' fields like locations with appended numbers or hyphens .. maybe.
6448         if (!empty($pseudoConstantMetadata['element']) && $pseudoConstantMetadata['element'] == $field) {
6449           $order = str_replace($field, "`{$field}`", $order);
6450         }
6451       }
6452     }
6453
6454     $this->_fromClause = self::fromClause($this->_tables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6455     $this->_simpleFromClause = self::fromClause($this->_whereTables, NULL, NULL, $this->_primaryLocation, $this->_mode);
6456
6457     // The above code relies on crazy brittle string manipulation of a peculiarly-encoded ORDER BY
6458     // clause. But this magic helper which forgivingly reescapes ORDER BY.
6459     if ($order) {
6460       $order = CRM_Utils_Type::escape($order, 'MysqlOrderBy');
6461       return ' ORDER BY ' . $order;
6462     }
6463   }
6464
6465   /**
6466    * Convert a string of group IDs to a string of group labels.
6467    *
6468    * The original string may include duplicates and groups the user does not have
6469    * permission to see.
6470    *
6471    * @param CRM_Core_DAO $dao
6472    * @param string $val
6473    *
6474    * @return string
6475    */
6476   public function convertGroupIDStringToLabelString(&$dao, $val) {
6477     $groupIDs = explode(',', $val);
6478     // Note that groups that the user does not have permission to will be excluded (good).
6479     $groups = array_intersect_key(CRM_Core_PseudoConstant::group(), array_flip($groupIDs));
6480     return implode(', ', $groups);
6481
6482   }
6483
6484   /**
6485    * Set the qill and where properties for a field.
6486    *
6487    * This function is intended as a short-term function to encourage refactoring
6488    * & re-use - but really we should just have less special-casing.
6489    *
6490    * @param string $name
6491    * @param string $op
6492    * @param string|array $value
6493    * @param string $grouping
6494    * @param string $field
6495    */
6496   public function setQillAndWhere($name, $op, $value, $grouping, $field) {
6497     $this->_where[$grouping][] = self::buildClause("contact_a.{$name}", $op, $value);
6498     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
6499     $this->_qill[$grouping][] = ts("%1 %2 %3", [
6500       1 => $field['title'],
6501       2 => $qillop,
6502       3 => $qillVal,
6503     ]);
6504   }
6505
6506   /**
6507    * Has the pseudoconstant of the field been requested.
6508    *
6509    * For example if the field is payment_instrument_id then it
6510    * has been requested if either payment_instrument_id or payment_instrument
6511    * have been requested. Payment_instrument is the option groun name field value.
6512    *
6513    * @param array $field
6514    * @param string $fieldName
6515    *   The unique name of the field - ie. the one it will be aliased to in the query.
6516    *
6517    * @return bool
6518    */
6519   private function pseudoConstantNameIsInReturnProperties($field, $fieldName = NULL) {
6520     $realField = $this->getMetadataForRealField($fieldName);
6521     if (!isset($realField['pseudoconstant'])) {
6522       return FALSE;
6523     }
6524     $pseudoConstant = $realField['pseudoconstant'];
6525     if (empty($pseudoConstant['optionGroupName']) &&
6526       CRM_Utils_Array::value('labelColumn', $pseudoConstant) !== 'name') {
6527       // We are increasing our pseudoconstant handling - but still very cautiously,
6528       // hence the check for labelColumn === name
6529       return FALSE;
6530     }
6531
6532     if (!empty($pseudoConstant['optionGroupName']) && CRM_Utils_Array::value($pseudoConstant['optionGroupName'], $this->_returnProperties)) {
6533       return TRUE;
6534     }
6535     if (CRM_Utils_Array::value($fieldName, $this->_returnProperties)) {
6536       return TRUE;
6537     }
6538     // Is this still required - the above goes off the unique name. Test with things like
6539     // communication_preferences & prefix_id.
6540     if (CRM_Utils_Array::value($field['name'], $this->_returnProperties)) {
6541       return TRUE;
6542     }
6543     return FALSE;
6544   }
6545
6546   /**
6547    * Get Select Clause.
6548    *
6549    * @return string
6550    */
6551   public function getSelect() {
6552     $select = "SELECT ";
6553     if (isset($this->_distinctComponentClause)) {
6554       $select .= "{$this->_distinctComponentClause}, ";
6555     }
6556     $select .= implode(', ', $this->_select);
6557     return $select;
6558   }
6559
6560   /**
6561    * Add basic statistics to the summary.
6562    *
6563    * @param array $summary
6564    * @param string $where
6565    * @param string $from
6566    *
6567    *
6568    * @return array
6569    * @throws \CRM_Core_Exception
6570    */
6571   protected function addBasicStatsToSummary(&$summary, $where, $from) {
6572     $summary['total']['count'] = 0;
6573     $summary['total']['amount'] = $summary['total']['avg'] = [];
6574
6575     $query = "
6576       SELECT COUNT( conts.total_amount ) as total_count,
6577         SUM(   conts.total_amount ) as total_amount,
6578         AVG(   conts.total_amount ) as total_avg,
6579         conts.currency              as currency
6580       FROM (
6581         SELECT civicrm_contribution.total_amount, COUNT(civicrm_contribution.total_amount) as civicrm_contribution_total_amount_count,
6582         civicrm_contribution.currency
6583         $from
6584         $where AND civicrm_contribution.contribution_status_id = 1
6585       GROUP BY civicrm_contribution.id
6586     ) as conts
6587     GROUP BY currency";
6588
6589     $dao = CRM_Core_DAO::executeQuery($query);
6590
6591     while ($dao->fetch()) {
6592       $summary['total']['count'] += $dao->total_count;
6593       $summary['total']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6594       $summary['total']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6595     }
6596
6597     if (!empty($summary['total']['amount'])) {
6598       $summary['total']['amount'] = implode(',&nbsp;', $summary['total']['amount']);
6599       $summary['total']['avg'] = implode(',&nbsp;', $summary['total']['avg']);
6600     }
6601     else {
6602       $summary['total']['amount'] = $summary['total']['avg'] = 0;
6603     }
6604     return $summary;
6605   }
6606
6607   /**
6608    * Add basic soft credit statistics to summary array.
6609    *
6610    * @param array $summary
6611    * @param string $where
6612    * @param string $from
6613    */
6614   protected function addBasicSoftCreditStatsToStats(&$summary, $where, $from) {
6615     $query = "
6616       SELECT COUNT( conts.total_amount ) as total_count,
6617       SUM(   conts.total_amount ) as total_amount,
6618       AVG(   conts.total_amount ) as total_avg,
6619       conts.currency as currency
6620       FROM (
6621         SELECT civicrm_contribution_soft.amount as total_amount, civicrm_contribution_soft.currency
6622         $from
6623         $where AND civicrm_contribution.contribution_status_id = 1 AND civicrm_contribution_soft.id IS NOT NULL
6624         GROUP BY civicrm_contribution_soft.id
6625       ) as conts
6626       GROUP BY currency";
6627
6628     $dao = CRM_Core_DAO::executeQuery($query);
6629     $summary['soft_credit']['count'] = 0;
6630     $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = [];
6631     while ($dao->fetch()) {
6632       $summary['soft_credit']['count'] += $dao->total_count;
6633       $summary['soft_credit']['amount'][] = CRM_Utils_Money::format($dao->total_amount, $dao->currency);
6634       $summary['soft_credit']['avg'][] = CRM_Utils_Money::format($dao->total_avg, $dao->currency);
6635     }
6636     if (!empty($summary['soft_credit']['amount'])) {
6637       $summary['soft_credit']['amount'] = implode(',&nbsp;', $summary['soft_credit']['amount']);
6638       $summary['soft_credit']['avg'] = implode(',&nbsp;', $summary['soft_credit']['avg']);
6639     }
6640     else {
6641       $summary['soft_credit']['amount'] = $summary['soft_credit']['avg'] = 0;
6642     }
6643   }
6644
6645   /**
6646    * Add basic stats about cancelled contributions to the summary.
6647    *
6648    * @param array $summary
6649    * @param string $where
6650    * @param string $from
6651    */
6652   protected function addBasicCancelStatsToSummary(&$summary, $where, $from) {
6653     $query = "
6654       SELECT COUNT( conts.total_amount ) as cancel_count,
6655        SUM(   conts.total_amount ) as cancel_amount,
6656        AVG(   conts.total_amount ) as cancel_avg,
6657        conts.currency              as currency
6658         FROM (
6659       SELECT civicrm_contribution.total_amount, civicrm_contribution.currency
6660       $from
6661       $where  AND civicrm_contribution.cancel_date IS NOT NULL
6662       GROUP BY civicrm_contribution.id
6663     ) as conts
6664     GROUP BY currency";
6665
6666     $dao = CRM_Core_DAO::executeQuery($query);
6667
6668     if ($dao->N <= 1) {
6669       if ($dao->fetch()) {
6670         $summary['cancel']['count'] = $dao->cancel_count;
6671         $summary['cancel']['amount'] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6672         $summary['cancel']['avg'] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6673       }
6674     }
6675     else {
6676       $summary['cancel']['count'] = 0;
6677       $summary['cancel']['amount'] = $summary['cancel']['avg'] = [];
6678       while ($dao->fetch()) {
6679         $summary['cancel']['count'] += $dao->cancel_count;
6680         $summary['cancel']['amount'][] = CRM_Utils_Money::format($dao->cancel_amount, $dao->currency);
6681         $summary['cancel']['avg'][] = CRM_Utils_Money::format($dao->cancel_avg, $dao->currency);
6682       }
6683       $summary['cancel']['amount'] = implode(',&nbsp;', $summary['cancel']['amount']);
6684       $summary['cancel']['avg'] = implode(',&nbsp;', $summary['cancel']['avg']);
6685     }
6686   }
6687
6688   /**
6689    * Create the sql query for an contact search.
6690    *
6691    * @param int $offset
6692    *   The offset for the query.
6693    * @param int $rowCount
6694    *   The number of rows to return.
6695    * @param string|CRM_Utils_Sort $sort
6696    *   The order by string.
6697    * @param bool $count
6698    *   Is this a count only query ?.
6699    * @param bool $includeContactIds
6700    *   Should we include contact ids?.
6701    * @param bool $sortByChar
6702    *   If true returns the distinct array of first characters for search results.
6703    * @param bool $groupContacts
6704    *   If true, return only the contact ids.
6705    * @param string $additionalWhereClause
6706    *   If the caller wants to further restrict the search (used for components).
6707    * @param null $sortOrder
6708    * @param string $additionalFromClause
6709    *   Should be clause with proper joins, effective to reduce where clause load.
6710    *
6711    * @param bool $skipOrderAndLimit
6712    * @return string
6713    */
6714   public function getSearchSQL(
6715     $offset = 0, $rowCount = 0, $sort = NULL,
6716     $count = FALSE, $includeContactIds = FALSE,
6717     $sortByChar = FALSE, $groupContacts = FALSE,
6718     $additionalWhereClause = NULL, $sortOrder = NULL,
6719     $additionalFromClause = NULL, $skipOrderAndLimit = FALSE) {
6720
6721     $sqlParts = $this->getSearchSQLParts($offset, $rowCount, $sort, $count, $includeContactIds, $sortByChar, $groupContacts, $additionalWhereClause, $sortOrder, $additionalFromClause);
6722
6723     if ($sortByChar) {
6724       CRM_Core_Error::deprecatedFunctionWarning('sort by char is deprecated - use alphabetQuery method');
6725       $sqlParts['order_by'] = 'ORDER BY sort_name asc';
6726     }
6727
6728     if ($skipOrderAndLimit) {
6729       CRM_Core_Error::deprecatedFunctionWarning('skipOrderAndLimit is deprected - call getSearchSQLParts & construct it in the calling function');
6730       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}";
6731     }
6732     else {
6733       $query = "{$sqlParts['select']} {$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']} {$sqlParts['order_by']} {$sqlParts['limit']}";
6734     }
6735     return $query;
6736   }
6737
6738   /**
6739    * Get the component parts of the search query as an array.
6740    *
6741    * @param int $offset
6742    *   The offset for the query.
6743    * @param int $rowCount
6744    *   The number of rows to return.
6745    * @param string|CRM_Utils_Sort $sort
6746    *   The order by string.
6747    * @param bool $count
6748    *   Is this a count only query ?.
6749    * @param bool $includeContactIds
6750    *   Should we include contact ids?.
6751    * @param bool $sortByChar
6752    *   If true returns the distinct array of first characters for search results.
6753    * @param bool $groupContacts
6754    *   If true, return only the contact ids.
6755    * @param string $additionalWhereClause
6756    *   If the caller wants to further restrict the search (used for components).
6757    * @param null $sortOrder
6758    * @param string $additionalFromClause
6759    *   Should be clause with proper joins, effective to reduce where clause load.
6760    *
6761    * @return array
6762    */
6763   public function getSearchSQLParts($offset = 0, $rowCount = 0, $sort = NULL,
6764     $count = FALSE, $includeContactIds = FALSE,
6765     $sortByChar = FALSE, $groupContacts = FALSE,
6766     $additionalWhereClause = NULL, $sortOrder = NULL,
6767     $additionalFromClause = NULL) {
6768     if ($includeContactIds) {
6769       $this->_includeContactIds = TRUE;
6770       $this->_whereClause = $this->whereClause();
6771     }
6772     $onlyDeleted = in_array([
6773       'deleted_contacts',
6774       '=',
6775       '1',
6776       '0',
6777       '0',
6778     ], $this->_params);
6779
6780     // if we’re explicitly looking for a certain contact’s contribs, events, etc.
6781     // and that contact happens to be deleted, set $onlyDeleted to true
6782     foreach ($this->_params as $values) {
6783       $name = CRM_Utils_Array::value(0, $values);
6784       $op = CRM_Utils_Array::value(1, $values);
6785       $value = CRM_Utils_Array::value(2, $values);
6786       if ($name == 'contact_id' and $op == '=') {
6787         if (CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $value, 'is_deleted')) {
6788           $onlyDeleted = TRUE;
6789         }
6790         break;
6791       }
6792     }
6793
6794     // building the query string
6795     $groupBy = $groupByCols = NULL;
6796     if (!$count) {
6797       if (isset($this->_groupByComponentClause)) {
6798         $groupByCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
6799         $groupByCols = explode(', ', $groupByCols);
6800       }
6801       elseif ($this->_useGroupBy) {
6802         $groupByCols = ['contact_a.id'];
6803       }
6804     }
6805     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
6806       $groupByCols = ['civicrm_activity.id'];
6807     }
6808     if (!empty($groupByCols)) {
6809       $groupBy = " GROUP BY " . implode(', ', $groupByCols);
6810     }
6811
6812     $order = $orderBy = '';
6813     if (!$count) {
6814       if (!$sortByChar) {
6815         $order = $this->prepareOrderBy($sort, $sortOrder);
6816       }
6817     }
6818     // Cases where we are disabling FGB (FULL_GROUP_BY_MODE):
6819     //   1. When GROUP BY columns are present then disable FGB otherwise it demands to add ORDER BY columns in GROUP BY and eventually in SELECT
6820     //     clause. This will impact the search query output.
6821     $disableFullGroupByMode = (!empty($groupBy) || $groupContacts);
6822
6823     if ($disableFullGroupByMode) {
6824       CRM_Core_DAO::disableFullGroupByMode();
6825     }
6826
6827     // CRM-15231
6828     $this->_sort = $sort;
6829
6830     //CRM-15967
6831     $this->includePseudoFieldsJoin($sort);
6832
6833     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
6834
6835     if ($additionalWhereClause) {
6836       $where = $where . ' AND ' . $additionalWhereClause;
6837     }
6838
6839     //additional from clause should be w/ proper joins.
6840     if ($additionalFromClause) {
6841       $from .= "\n" . $additionalFromClause;
6842     }
6843
6844     // if we are doing a transform, do it here
6845     // use the $from, $where and $having to get the contact ID
6846     if ($this->_displayRelationshipType) {
6847       $this->filterRelatedContacts($from, $where, $having);
6848     }
6849     $limit = (!$count && $rowCount) ? " LIMIT " . CRM_Utils_Type::escape($offset, 'Int') . ", " . CRM_Utils_Type::escape($rowCount, 'Int') : '';
6850
6851     return [
6852       'select' => $select,
6853       'from' => $from,
6854       'where' => $where,
6855       'order_by' => $order,
6856       'group_by' => $groupBy,
6857       'having' => $having,
6858       'limit' => $limit,
6859     ];
6860   }
6861
6862   /**
6863    * Get the metadata for a given field.
6864    *
6865    * @param string $fieldName
6866    *
6867    * @return array
6868    */
6869   protected function getMetadataForField($fieldName) {
6870     if ($fieldName === 'contact_a.id') {
6871       // This seems to be the only anomaly.
6872       $fieldName = 'id';
6873     }
6874     $pseudoField = isset($this->_pseudoConstantsSelect[$fieldName]) ? $this->_pseudoConstantsSelect[$fieldName] : [];
6875     $field = isset($this->_fields[$fieldName]) ? $this->_fields[$fieldName] : $pseudoField;
6876     $field = array_merge($field, $pseudoField);
6877     if (!empty($field) && empty($field['name'])) {
6878       // standardising field formatting here - over time we can phase out variants.
6879       // all paths using this currently unit tested
6880       $field['name'] = CRM_Utils_Array::value('field_name', $field, CRM_Utils_Array::value('idCol', $field, $fieldName));
6881     }
6882     return $field;
6883   }
6884
6885   /**
6886    * Get the metadata for a given field, returning the 'real field' if it is a pseudofield.
6887    *
6888    * @param string $fieldName
6889    *
6890    * @return array
6891    */
6892   protected function getMetadataForRealField($fieldName) {
6893     $field = $this->getMetadataForField($fieldName);
6894     if (!empty($field['is_pseudofield_for'])) {
6895       $field = $this->getMetadataForField($field['is_pseudofield_for']);
6896       $field['pseudofield_name'] = $fieldName;
6897     }
6898     elseif (!empty($field['pseudoconstant'])) {
6899       if (!empty($field['pseudoconstant']['optionGroupName'])) {
6900         $field['pseudofield_name'] = $field['pseudoconstant']['optionGroupName'];
6901         if (empty($field['table_name'])) {
6902           if (!empty($field['where'])) {
6903             $field['table_name'] = explode('.', $field['where'])[0];
6904           }
6905           else {
6906             $field['table_name'] = 'civicrm_contact';
6907           }
6908         }
6909       }
6910     }
6911     return $field;
6912   }
6913
6914   /**
6915    * If we have a field that is better rendered via the pseudoconstant handled them here.
6916    *
6917    * Rather than joining in the additional table we render the option value on output.
6918    *
6919    * @todo - so far this applies to a narrow range of pseudocontants. We are adding them
6920    * carefully with test coverage but aim to extend.
6921    *
6922    * @param string $name
6923    */
6924   protected function addPseudoconstantFieldToSelect($name) {
6925     $field = $this->getMetadataForRealField($name);
6926     $realFieldName = $field['name'];
6927     $pseudoFieldName = CRM_Utils_Array::value('pseudofield_name', $field);
6928     if ($pseudoFieldName) {
6929       // @todo - we don't really need to build this array now we have metadata more available with getMetadataForField fn.
6930       $this->_pseudoConstantsSelect[$pseudoFieldName] = [
6931         'pseudoField' => $pseudoFieldName,
6932         'idCol' => $realFieldName,
6933         'field_name' => $field['name'],
6934         'bao' => $field['bao'],
6935         'pseudoconstant' => $field['pseudoconstant'],
6936       ];
6937     }
6938
6939     $this->_tables[$field['table_name']] = 1;
6940     $this->_element[$realFieldName] = 1;
6941     $this->_select[$field['name']] = str_replace('civicrm_contact.', 'contact_a.', "{$field['where']} as `$realFieldName`");
6942   }
6943
6944   /**
6945    * Is this pseudofield a foreign key constraint.
6946    *
6947    * We are trying to cautiously expand our pseudoconstant handling. This check allows us
6948    * to extend to a narrowly defined type (and then only if the pseudofield is in the fields
6949    * array which is done for contributions which are mostly handled as pseudoconstants.
6950    *
6951    * @param $fieldSpec
6952    *
6953    * @return bool
6954    */
6955   protected function isPseudoFieldAnFK($fieldSpec) {
6956     if (empty($fieldSpec['FKClassName'])
6957       || CRM_Utils_Array::value('keyColumn', $fieldSpec['pseudoconstant']) !== 'id'
6958       || CRM_Utils_Array::value('labelColumn', $fieldSpec['pseudoconstant']) !== 'name') {
6959       return FALSE;
6960     }
6961     return TRUE;
6962   }
6963
6964   /**
6965    * Is the field a relative date field.
6966    *
6967    * @param string $fieldName
6968    *
6969    * @return bool
6970    */
6971   protected function isARelativeDateField($fieldName) {
6972     if (substr($fieldName, -9, 9) !== '_relative') {
6973       return FALSE;
6974     }
6975     $realField = substr($fieldName, 0, strlen($fieldName) - 9);
6976     return isset($this->_fields[$realField]);
6977   }
6978
6979   /**
6980    * Get the specifications for the field, if available.
6981    *
6982    * @param string $fieldName
6983    *   Fieldname as displayed on the form.
6984    *
6985    * @return array
6986    */
6987   public function getFieldSpec($fieldName) {
6988     if (isset($this->_fields[$fieldName])) {
6989       return $this->_fields[$fieldName];
6990     }
6991     $lowFieldName = str_replace('_low', '', $fieldName);
6992     if (isset($this->_fields[$lowFieldName])) {
6993       return array_merge($this->_fields[$lowFieldName], ['field_name' => $lowFieldName]);
6994     }
6995     $highFieldName = str_replace('_high', '', $fieldName);
6996     if (isset($this->_fields[$highFieldName])) {
6997       return array_merge($this->_fields[$highFieldName], ['field_name' => $highFieldName]);
6998     }
6999     return [];
7000   }
7001
7002   public function buildWhereForDate() {
7003
7004   }
7005
7006   /**
7007    * Is the field a relative date field.
7008    *
7009    * @param string $fieldName
7010    *
7011    * @return bool
7012    */
7013   protected function isADateRangeField($fieldName) {
7014     if (substr($fieldName, -4, 4) !== '_low' && substr($fieldName, -5, 5) !== '_high') {
7015       return FALSE;
7016     }
7017     return !empty($this->getFieldSpec($fieldName));
7018   }
7019
7020   /**
7021    * @param $values
7022    */
7023   protected function buildRelativeDateQuery(&$values) {
7024     $value = CRM_Utils_Array::value(2, $values);
7025     if (empty($value)) {
7026       return;
7027     }
7028     $fieldName = substr($values[0], 0, strlen($values[0]) - 9);
7029     $fieldSpec = $this->_fields[$fieldName];
7030     $tableName = $fieldSpec['table_name'];
7031     $filters = CRM_Core_OptionGroup::values('relative_date_filters');
7032     $grouping = CRM_Utils_Array::value(3, $values);
7033     // If the table value is already set for a custom field it will be more nuanced than just '1'.
7034     $this->_tables[$tableName] = $this->_tables[$tableName] ?? 1;
7035     $this->_whereTables[$tableName] = $this->_whereTables[$tableName] ?? 1;
7036
7037     $dates = CRM_Utils_Date::getFromTo($value, NULL, NULL);
7038     if (empty($dates[0])) {
7039       // ie. no start date we only have end date
7040       $this->_where[$grouping][] = $fieldSpec['where'] . " <= '{$dates[1]}'";
7041
7042       $this->_qill[$grouping][] = ts('%1 is ', [$fieldSpec['title']]) . $filters[$value] . ' (' . ts("to %1", [
7043         CRM_Utils_Date::customFormat($dates[1]),
7044       ]) . ')';
7045     }
7046     elseif (empty($dates[1])) {
7047       // ie. no end date we only have start date
7048       $this->_where[$grouping][] = $fieldSpec['where'] . " >= '{$dates[1]}'";
7049
7050       $this->_qill[$grouping][] = ts('%1 is ', [$fieldSpec['title']]) . $filters[$value] . ' (' . ts("from %1", [
7051         CRM_Utils_Date::customFormat($dates[0]),
7052       ]) . ')';
7053     }
7054     else {
7055       // we have start and end dates.
7056       $this->_where[$grouping][] = $fieldSpec['where'] . " BETWEEN '{$dates[0]}' AND '{$dates[1]}'";
7057
7058       $this->_qill[$grouping][] = ts('%1 is ', [$fieldSpec['title']]) . $filters[$value] . ' (' . ts("between %1 and %2", [
7059         CRM_Utils_Date::customFormat($dates[0]),
7060         CRM_Utils_Date::customFormat($dates[1]),
7061       ]) . ')';
7062     }
7063   }
7064
7065   /**
7066    * Build the query for a date field if it is a _high or _low field.
7067    *
7068    * @param $values
7069    *
7070    * @return bool
7071    */
7072   public function buildDateRangeQuery($values) {
7073     if ($this->isADateRangeField($values[0])) {
7074       $fieldSpec = $this->getFieldSpec($values[0]);
7075       $title = empty($fieldSpec['unique_title']) ? $fieldSpec['title'] : $fieldSpec['unique_title'];
7076       $this->dateQueryBuilder($values, $fieldSpec['table_name'], $fieldSpec['field_name'], $fieldSpec['name'], $title);
7077       return TRUE;
7078     }
7079     return FALSE;
7080   }
7081
7082   /**
7083    * Add the address table into the query.
7084    *
7085    * @param string $tableKey
7086    * @param string $joinCondition
7087    *
7088    * @return array
7089    *   - alias name
7090    *   - address join.
7091    */
7092   protected function addAddressTable($tableKey, $joinCondition) {
7093     $tName = "$tableKey-address";
7094     $aName = "`$tableKey-address`";
7095     $this->_select["{$tName}_id"] = "`$tName`.id as `{$tName}_id`";
7096     $this->_element["{$tName}_id"] = 1;
7097     $addressJoin = "\nLEFT JOIN civicrm_address $aName ON ($aName.contact_id = contact_a.id AND $aName.$joinCondition)";
7098     $this->_tables[$tName] = $addressJoin;
7099
7100     return [
7101       $aName,
7102       $addressJoin,
7103     ];
7104   }
7105
7106   /**
7107    * Get the clause for group status.
7108    *
7109    * @param int $grouping
7110    *
7111    * @return string
7112    */
7113   protected function getGroupStatusClause($grouping) {
7114     $statuses = $this->getSelectedGroupStatuses($grouping);
7115     return "status IN (" . implode(', ', $statuses) . ")";
7116   }
7117
7118   /**
7119    * Get an array of the statuses that have been selected.
7120    *
7121    * @param string $grouping
7122    *
7123    * @return array
7124    */
7125   protected function getSelectedGroupStatuses($grouping) {
7126     $statuses = [];
7127     $gcsValues = $this->getWhereValues('group_contact_status', $grouping);
7128     if ($gcsValues &&
7129       is_array($gcsValues[2])
7130     ) {
7131       foreach ($gcsValues[2] as $k => $v) {
7132         if ($v) {
7133           $statuses[] = "'" . CRM_Utils_Type::escape($k, 'String') . "'";
7134         }
7135       }
7136     }
7137     else {
7138       $statuses[] = "'Added'";
7139     }
7140     return $statuses;
7141   }
7142
7143   /**
7144    * Get the qill value for the field.
7145    *
7146    * @param $name
7147    * @param array|int|string $value
7148    * @param $op
7149    *
7150    * @return string
7151    */
7152   protected function getQillForField($name, $value, $op): string {
7153     list($qillop, $qillVal) = CRM_Contact_BAO_Query::buildQillForFieldValue(NULL, $name, $value, $op);
7154     return (string) ts("%1 %2 %3", [
7155       1 => ts('Email'),
7156       2 => $qillop,
7157       3 => $qillVal,
7158     ]);
7159   }
7160
7161 }