3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
30 * @copyright CiviCRM LLC (c) 2004-2019
34 * Class CRM_Utils_QueryFormatter
36 * This class is a bad idea. It exists for the unholy reason that a single installation
37 * may have up to three query engines (MySQL LIKE, MySQL FTS, Solr) processing the same
38 * query-text. It labors* to take the user's search expression and provide similar search
39 * semantics in different contexts. It is unknown whether this labor will be fruitful
42 class CRM_Utils_QueryFormatter
{
44 * Generate queries using SQL LIKE expressions.
46 const LANG_SQL_LIKE
= 'like';
49 * Generate queries using MySQL FTS expressions.
51 const LANG_SQL_FTS
= 'fts';
54 * Generate queries using MySQL's boolean FTS expressions.
56 const LANG_SQL_FTSBOOL
= 'ftsbool';
59 * Generate queries using Solr expressions.
61 const LANG_SOLR
= 'solr';
64 * Attempt to leave the text as-is.
66 const MODE_NONE
= 'simple';
69 * Attempt to treat the input text as a phrase
71 const MODE_PHRASE
= 'phrase';
74 * Attempt to treat the input text as a phrase with
75 * wildcards on each end.
77 const MODE_WILDPHRASE
= 'wildphrase';
80 * Attempt to treat individual word as if it
81 * had wildcards at the start and end.
83 const MODE_WILDWORDS
= 'wildwords';
86 * Attempt to treat individual word as if it
87 * had a wildcard at the end.
89 const MODE_WILDWORDS_SUFFIX
= 'wildwords-suffix';
94 * @var \CRM_Utils_QueryFormatter|null
96 static protected $singleton;
100 * @return CRM_Utils_QueryFormatter
102 public static function singleton($fresh = FALSE) {
103 if ($fresh || self
::$singleton === NULL) {
104 $mode = Civi
::settings()->get('fts_query_mode');
105 self
::$singleton = new CRM_Utils_QueryFormatter($mode);
107 return self
::$singleton;
117 * @param string $mode
120 public function __construct($mode) {
127 public function setMode($mode) {
134 public function getMode() {
139 * @param string $text
140 * @param string $language
141 * Eg LANG_SQL_LIKE, LANG_SQL_FTS, LANG_SOLR.
142 * @throws CRM_Core_Exception
145 public function format($text, $language) {
149 case self
::LANG_SOLR
:
150 case self
::LANG_SQL_FTS
:
151 $text = $this->_formatFts($text, $this->mode
);
154 case self
::LANG_SQL_FTSBOOL
:
155 $text = $this->_formatFtsBool($text, $this->mode
);
158 case self
::LANG_SQL_LIKE
:
159 $text = $this->_formatLike($text, $this->mode
);
166 if ($text === NULL) {
167 throw new CRM_Core_Exception("Unrecognized combination: language=[{$language}] mode=[{$this->mode}]");
174 * Create a SQL WHERE expression for matching against a list of
177 * @param string $table
178 * Eg "civicrm_note" or "civicrm_note mynote".
179 * @param array|string $columns
180 * List of columns to search against.
181 * Eg "first_name" or "activity_details".
182 * @param string $queryText
184 * SQL, eg "MATCH (col1) AGAINST (queryText)" or "col1 LIKE '%queryText%'"
186 public function formatSql($table, $columns, $queryText) {
187 if ($queryText === '*' ||
$queryText === '%' ||
empty($queryText)) {
191 $strtolower = function_exists('mb_strtolower') ?
'mb_strtolower' : 'strtolower';
193 if (strpos($table, ' ') === FALSE) {
194 $tableName = $tableAlias = $table;
197 list ($tableName, $tableAlias) = explode(' ', $table);
199 if (is_scalar($columns)) {
200 $columns = [$columns];
204 if (CRM_Core_InnoDBIndexer
::singleton()
205 ->hasDeclaredIndex($tableName, $columns)
207 $formattedQuery = $this->format($queryText, CRM_Utils_QueryFormatter
::LANG_SQL_FTSBOOL
);
209 $prefixedFieldNames = [];
210 foreach ($columns as $fieldName) {
211 $prefixedFieldNames[] = "$tableAlias.$fieldName";
214 $clauses[] = sprintf("MATCH (%s) AGAINST ('%s' IN BOOLEAN MODE)",
215 implode(',', $prefixedFieldNames),
216 $strtolower(CRM_Core_DAO
::escapeString($formattedQuery))
220 //CRM_Core_Session::setStatus(ts('Cannot use FTS for %1 (%2)', array(
222 // 2 => implode(', ', $fullTextFields),
225 $formattedQuery = $this->format($queryText, CRM_Utils_QueryFormatter
::LANG_SQL_LIKE
);
226 $escapedText = $strtolower(CRM_Core_DAO
::escapeString($formattedQuery));
227 foreach ($columns as $fieldName) {
228 $clauses[] = "$tableAlias.$fieldName LIKE '{$escapedText}'";
231 return implode(' OR ', $clauses);
237 * @param string $text
242 protected function _formatFts($text, $mode) {
245 // normalize user-inputted wildcards
246 $text = str_replace('%', '*', $text);
251 elseif (strpos($text, '*') !== FALSE) {
252 // if user supplies their own wildcards, then don't do any sophisticated changes
257 case self
::MODE_NONE
:
261 case self
::MODE_PHRASE
:
262 $result = '"' . $text . '"';
265 case self
::MODE_WILDPHRASE
:
266 $result = '"*' . $text . '*"';
269 case self
::MODE_WILDWORDS
:
270 $result = $this->mapWords($text, '*word*');
273 case self
::MODE_WILDWORDS_SUFFIX
:
274 $result = $this->mapWords($text, 'word*');
282 return $this->dedupeWildcards($result, '%');
288 * @param string $text
293 protected function _formatFtsBool($text, $mode) {
295 $operators = ['+', '-', '~', '(', ')'];
296 $wildCards = ['@', '%', '*'];
297 $expression = preg_quote(implode('', array_merge($operators, $wildCards)), '/');
299 //Return if searched string ends with an unsupported operator.
300 //Or if the string contains an invalid joint occurrence of operators.
301 foreach ($operators as $val) {
302 if ($text == '@' || CRM_Utils_String
::endsWith($text, $val) ||
preg_match("/[{$expression}]{2,}/", $text)) {
303 $csid = CRM_Core_DAO
::getFieldValue('CRM_Core_DAO_OptionValue', 'CRM_Contact_Form_Search_Custom_FullText', 'value', 'name');
304 $url = CRM_Utils_System
::url("civicrm/contact/search/custom", "csid={$csid}&reset=1");
305 $operators = implode("', '", $operators);
306 CRM_Core_Error
::statusBounce("Full-Text Search does not support the use of a search with two attached operators or string ending with any of these operators ('{$operators}' or a single '@'). Please adjust your search term and try again.", $url, 'Invalid Search String');
310 // normalize user-inputted wildcards
311 $text = str_replace('%', '*', $text);
316 elseif (strpos($text, '+') !== FALSE ||
strpos($text, '-') !== FALSE) {
317 // if user supplies their own include/exclude operators, use text as is (with trailing wildcard)
318 $result = $this->mapWords($text, 'word*');
320 elseif (strpos($text, '*') !== FALSE) {
321 // if user supplies their own wildcards, then don't do any sophisticated changes
322 $result = $this->mapWords($text, '+word');
324 elseif (preg_match('/^(["\']).*\1$/m', $text)) {
325 // if surrounded by quotes, use term as is
330 case self
::MODE_NONE
:
331 $result = $this->mapWords($text, '+word', TRUE);
334 case self
::MODE_PHRASE
:
335 $result = '+"' . $text . '"';
338 case self
::MODE_WILDPHRASE
:
339 $result = '+"*' . $text . '*"';
342 case self
::MODE_WILDWORDS
:
343 $result = $this->mapWords($text, '+*word*');
346 case self
::MODE_WILDWORDS_SUFFIX
:
347 $result = $this->mapWords($text, '+word*');
355 return $this->dedupeWildcards($result, '%');
366 protected function _formatLike($text, $mode) {
372 elseif (strpos($text, '%') !== FALSE) {
373 // if user supplies their own wildcards, then don't do any sophisticated changes
378 case self
::MODE_NONE
:
379 case self
::MODE_PHRASE
:
380 case self
::MODE_WILDPHRASE
:
381 $result = "%" . $text . "%";
384 case self
::MODE_WILDWORDS
:
385 case self
::MODE_WILDWORDS_SUFFIX
:
386 $result = "%" . preg_replace('/[ \r\n]+/', '%', $text) . '%';
394 return $this->dedupeWildcards($result, '%');
398 * @param string $text
399 * User-supplied query string.
400 * @param string $template
401 * A prototypical description of each word, eg "word%" or "word*" or "*word*".
402 * @param bool $quotes
403 * True if each searched keyword need to be surrounded with quotes.
406 protected function mapWords($text, $template, $quotes = FALSE) {
408 foreach ($this->parseWords($text, $quotes) as $word) {
409 $result[] = str_replace('word', $word, $template);
411 return implode(' ', $result);
415 * @param string $text
416 * @param bool $quotes
419 protected function parseWords($text, $quotes) {
420 //NYSS 9692 special handling for emails
421 if (preg_match('/^([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})$/', $text)) {
422 $parts = explode('@', $text);
423 $parts[1] = stristr($parts[1], '.', TRUE);
424 $text = implode(' ', $parts);
427 //NYSS also replace other occurrences of @
428 $replacedText = preg_replace('/[ \r\n\t\@]+/', ' ', trim($text));
429 //filter empty values if any
430 $keywords = array_filter(explode(' ', $replacedText));
432 //Ensure each searched keywords are wrapped in double quotes.
434 foreach ($keywords as &$val) {
435 if (!is_numeric($val)) {
448 protected function dedupeWildcards($text, $wildcard) {
449 if ($text === NULL) {
453 // don't use preg_replace because $wildcard might be special char
454 while (strpos($text, "{$wildcard}{$wildcard}") !== FALSE) {
455 $text = str_replace("{$wildcard}{$wildcard}", "{$wildcard}", $text);
465 public static function getModes() {
469 self
::MODE_WILDPHRASE
,
470 self
::MODE_WILDWORDS
,
471 self
::MODE_WILDWORDS_SUFFIX
,
480 public static function getLanguages() {
484 self
::LANG_SQL_FTSBOOL
,
492 * Ex: drush eval 'civicrm_initialize(); CRM_Utils_QueryFormatter::dumpExampleTable("firstword secondword");'
494 public static function dumpExampleTable($text) {
495 $width = strlen($text) +
8;
498 $buf .= sprintf("%-{$width}s", 'mode');
499 foreach (self
::getLanguages() as $lang) {
500 $buf .= sprintf("%-{$width}s", $lang);
504 foreach (self
::getModes() as $mode) {
505 $formatter = new CRM_Utils_QueryFormatter($mode);
506 $buf .= sprintf("%-{$width}s", $mode);
507 foreach (self
::getLanguages() as $lang) {
508 $buf .= sprintf("%-{$width}s", $formatter->format($text, $lang));