3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
17 class CRM_Contact_BAO_GroupContactCache
extends CRM_Contact_DAO_GroupContactCache
{
19 public static $_alreadyLoaded = [];
22 * Get a list of caching modes.
26 public static function getModes() {
28 // Flush expired caches in response to user actions.
29 'opportunistic' => ts('Opportunistic Flush'),
31 // Flush expired caches via background cron jobs.
32 'deterministic' => ts('Cron Flush'),
37 * Check to see if we have cache entries for this group.
39 * If not, regenerate, else return.
41 * @param array $groupIDs
42 * Of group that we are checking against.
45 * TRUE if we did not regenerate, FALSE if we did
47 public static function check($groupIDs) {
48 if (empty($groupIDs)) {
52 return self
::loadAll($groupIDs);
56 * Formulate the query to see which groups needs to be refreshed.
58 * The calculation is based on their cache date and the smartGroupCacheTimeOut
60 * @param string $groupIDClause
61 * The clause which limits which groups we need to evaluate.
62 * @param bool $includeHiddenGroups
63 * Hidden groups are excluded by default.
66 * the sql query which lists the groups that need to be refreshed
68 public static function groupRefreshedClause($groupIDClause = NULL, $includeHiddenGroups = FALSE) {
69 $smartGroupCacheTimeoutDateTime = self
::getCacheInvalidDateTime();
74 WHERE ( g.saved_search_id IS NOT NULL OR g.children IS NOT NULL )
78 OR cache_date <= $smartGroupCacheTimeoutDateTime
79 OR NOW() >= g.refresh_date
82 if (!$includeHiddenGroups) {
83 $query .= "AND (g.is_hidden = 0 OR g.is_hidden IS NULL)";
86 if (!empty($groupIDClause)) {
87 $query .= " AND ( $groupIDClause ) ";
94 * Check to see if a group has been refreshed recently.
96 * This is primarily used in a locking scenario when some other process might have refreshed things underneath
101 * @param bool $includeHiddenGroups
102 * Hidden groups are excluded by default.
105 * the sql query which lists the groups that need to be refreshed
107 public static function shouldGroupBeRefreshed($groupID, $includeHiddenGroups = FALSE) {
108 $query = self
::groupRefreshedClause("g.id = %1", $includeHiddenGroups);
109 $params = [1 => [$groupID, 'Integer']];
111 // if the query returns the group ID, it means the group is a valid candidate for refreshing
112 return CRM_Core_DAO
::singleValueQuery($query, $params);
116 * Check to see if we have cache entries for this group.
118 * if not, regenerate, else return
120 * @param int|array $groupIDs groupIDs of group that we are checking against
121 * if empty, all groups are checked
123 * Limits the number of groups we evaluate.
126 * TRUE if we did not regenerate, FALSE if we did
128 public static function loadAll($groupIDs = NULL, $limit = 0) {
129 // ensure that all the smart groups are loaded
130 // this function is expensive and should be sparingly used if groupIDs is empty
131 if (empty($groupIDs)) {
132 $groupIDClause = NULL;
136 if (!is_array($groupIDs)) {
137 $groupIDs = [$groupIDs];
140 // note escapeString is a must here and we can't send the imploded value as second argument to
141 // the executeQuery(), since that would put single quote around the string and such a string
142 // of comma separated integers would not work.
143 $groupIDString = CRM_Core_DAO
::escapeString(implode(', ', $groupIDs));
145 $groupIDClause = "g.id IN ({$groupIDString})";
148 $query = self
::groupRefreshedClause($groupIDClause);
150 $limitClause = $orderClause = NULL;
152 $limitClause = " LIMIT 0, $limit";
153 $orderClause = " ORDER BY g.cache_date, g.refresh_date";
155 // We ignore hidden groups and disabled groups
161 $dao = CRM_Core_DAO
::executeQuery($query);
162 $processGroupIDs = [];
163 $refreshGroupIDs = $groupIDs;
164 while ($dao->fetch()) {
165 $processGroupIDs[] = $dao->id
;
167 // remove this id from refreshGroupIDs
168 foreach ($refreshGroupIDs as $idx => $gid) {
169 if ($gid == $dao->id
) {
170 unset($refreshGroupIDs[$idx]);
176 if (!empty($refreshGroupIDs)) {
177 $refreshGroupIDString = CRM_Core_DAO
::escapeString(implode(', ', $refreshGroupIDs));
178 $time = self
::getRefreshDateTime();
180 UPDATE civicrm_group g
181 SET g.refresh_date = $time
182 WHERE g.id IN ( {$refreshGroupIDString} )
183 AND g.refresh_date IS NULL
185 CRM_Core_DAO
::executeQuery($query);
188 if (empty($processGroupIDs)) {
192 self
::add($processGroupIDs);
198 * Build the smart group cache for given groups.
200 * @param array $groupIDs
202 public static function add($groupIDs) {
203 $groupIDs = (array) $groupIDs;
205 foreach ($groupIDs as $groupID) {
206 // first delete the current cache
207 self
::clearGroupContactCache($groupID);
208 $params = [['group', 'IN', [$groupID], 0, 0]];
209 // the below call updates the cache table as a byproduct of the query
210 CRM_Contact_BAO_Query
::apiQuery($params, ['contact_id'], NULL, NULL, 0, 0, FALSE);
215 * Store values into the group contact cache.
217 * @todo review use of INSERT IGNORE. This function appears to be slower that inserting
218 * with a left join. Also, 200 at once seems too little.
220 * @param array $groupID
221 * @param array $values
223 public static function store($groupID, &$values) {
226 // sort the values so we put group IDs in front and hence optimize
227 // mysql storage (or so we think) CRM-9493
230 // to avoid long strings, lets do BULK_INSERT_COUNT values at a time
231 while (!empty($values)) {
233 $input = array_splice($values, 0, CRM_Core_DAO
::BULK_INSERT_COUNT
);
234 $str = implode(',', $input);
235 $sql = "INSERT IGNORE INTO civicrm_group_contact_cache (group_id,contact_id) VALUES $str;";
236 CRM_Core_DAO
::executeQuery($sql);
238 self
::updateCacheTime($groupID, $processed);
242 * Change the cache_date.
244 * @param array $groupID
245 * @param bool $processed
246 * Whether the cache data was recently modified.
248 public static function updateCacheTime($groupID, $processed) {
249 // only update cache entry if we had any values
251 // also update the group with cache date information
252 $now = date('YmdHis');
260 $groupIDs = implode(',', $groupID);
263 SET cache_date = $now, refresh_date = $refresh
264 WHERE id IN ( $groupIDs )
266 CRM_Core_DAO
::executeQuery($sql);
270 * @deprecated function - the best function to call is
271 * CRM_Contact_BAO_Contact::updateContactCache at the moment, or api job.group_cache_flush
272 * to really force a flush.
274 * Remove this function altogether by mid 2018.
276 * However, if updating code outside core to use this (or any BAO function) it is recommended that
277 * you add an api call to lock in into our contract. Currently there is not really a supported
278 * method for non core functions.
280 public static function remove() {
282 ->warning('Deprecated code. This function should not be called without groupIDs. Extensions can use the api job.group_cache_flush for a hard flush or add an api option for soft flush', ['civi.tag' => 'deprecated']);
283 CRM_Contact_BAO_GroupContactCache
::opportunisticCacheFlush();
287 * Function to clear group contact cache and reset the corresponding
288 * group's cache and refresh date
290 * @param int $groupID
293 public static function clearGroupContactCache($groupID) {
294 $transaction = new CRM_Core_Transaction();
297 FROM civicrm_group_contact_cache g
298 WHERE g.group_id = %1 ";
301 UPDATE civicrm_group g
302 SET cache_date = null, refresh_date = null
306 1 => [$groupID, 'Integer'],
309 CRM_Core_DAO
::executeQuery($query, $params);
310 // also update the cache_date for these groups
311 CRM_Core_DAO
::executeQuery($update, $params);
312 unset(self
::$_alreadyLoaded[$groupID]);
314 $transaction->commit();
318 * Refresh the smart group cache tables.
320 * This involves clearing out any aged entries (based on the site timeout setting) and resetting the time outs.
322 * This function should be called via the opportunistic or deterministic cache refresh function to make the intent
325 protected static function flushCaches() {
327 $lock = self
::getLockForRefresh();
329 catch (CRM_Core_Exception
$e) {
330 // Someone else is kindly doing the refresh for us right now.
333 $params = [1 => [self
::getCacheInvalidDateTime(), 'String']];
334 // @todo this is consistent with previous behaviour but as the first query could take several seconds the second
335 // could become inaccurate. It seems to make more sense to fetch them first & delete from an array (which would
336 // also reduce joins). If we do this we should also consider how best to iterate the groups. If we do them one at
337 // a time we could call a hook, allowing people to manage the frequency on their groups, or possibly custom searches
338 // might do that too. However, for 2000 groups that's 2000 iterations. If we do all once we potentially create a
339 // slow query. It's worth noting the speed issue generally relates to the size of the group but if one slow group
340 // is in a query with 500 fast ones all 500 get locked. One approach might be to calculate group size or the
341 // number of groups & then process all at once or many query runs depending on what is found. Of course those
342 // preliminary queries would need speed testing.
343 CRM_Core_DAO
::executeQuery(
346 FROM civicrm_group_contact_cache gc
347 INNER JOIN civicrm_group g ON g.id = gc.group_id
348 WHERE g.cache_date <= %1
353 // Clear these out without resetting them because we are not building caches here, only clearing them,
354 // so the state is 'as if they had never been built'.
355 CRM_Core_DAO
::executeQuery(
357 UPDATE civicrm_group g
358 SET cache_date = NULL,
360 WHERE g.cache_date <= %1
368 * Check if the refresh is already initiated.
370 * We have 2 imperfect methods for this:
371 * 1) a static variable in the function. This works fine within a request
372 * 2) a mysql lock. This works fine as long as CiviMail is not running, or if mysql is version 5.7+
374 * Where these 2 locks fail we get 2 processes running at the same time, but we have at least minimised that.
376 * @return \Civi\Core\Lock\LockInterface
377 * @throws \CRM_Core_Exception
379 protected static function getLockForRefresh() {
380 if (!isset(Civi
::$statics[__CLASS__
]['is_refresh_init'])) {
381 Civi
::$statics[__CLASS__
] = ['is_refresh_init' => FALSE];
384 if (Civi
::$statics[__CLASS__
]['is_refresh_init']) {
385 throw new CRM_Core_Exception('A refresh has already run in this process');
387 $lock = Civi
::lockManager()->acquire('data.core.group.refresh');
388 if ($lock->isAcquired()) {
389 Civi
::$statics[__CLASS__
]['is_refresh_init'] = TRUE;
392 throw new CRM_Core_Exception('Mysql lock unavailable');
396 * Do an opportunistic cache refresh if the site is configured for these.
398 * Sites that do not run the smart group clearing cron job should refresh the
399 * caches on demand. The user session will be forced to wait so it is less
402 public static function opportunisticCacheFlush() {
403 if (Civi
::settings()->get('smart_group_cache_refresh_mode') == 'opportunistic') {
409 * Do a forced cache refresh.
411 * This function is appropriate to be called by system jobs & non-user sessions.
413 public static function deterministicCacheFlush() {
414 if (self
::smartGroupCacheTimeout() == 0) {
415 CRM_Core_DAO
::executeQuery("TRUNCATE civicrm_group_contact_cache");
416 CRM_Core_DAO
::executeQuery("
417 UPDATE civicrm_group g
418 SET cache_date = null, refresh_date = null");
426 * Remove one or more contacts from the smart group cache.
428 * @param int|array $cid
429 * @param int $groupId
432 * TRUE if successful.
434 public static function removeContact($cid, $groupId = NULL) {
437 foreach ((array) $cid as $c) {
438 $cids[] = CRM_Utils_Type
::escape($c, 'Integer');
441 $condition = count($cids) == 1 ?
"= {$cids[0]}" : "IN (" . implode(',', $cids) . ")";
443 $condition .= " AND group_id = " . CRM_Utils_Type
::escape($groupId, 'Integer');
445 $sql = "DELETE FROM civicrm_group_contact_cache WHERE contact_id $condition";
446 CRM_Core_DAO
::executeQuery($sql);
453 * Load the smart group cache for a saved search.
455 * @param object $group
456 * The smart group that needs to be loaded.
458 * Should we force a search through.
460 * @throws \CRM_Core_Exception
462 public static function load(&$group, $force = FALSE) {
463 $groupID = $group->id
;
464 $savedSearchID = $group->saved_search_id
;
465 if (array_key_exists($groupID, self
::$_alreadyLoaded) && !$force) {
469 self
::$_alreadyLoaded[$groupID] = 1;
471 // FIXME: some other process could have actually done the work before we got here,
472 // Ensure that work needs to be done before continuing
473 if (!$force && !self
::shouldGroupBeRefreshed($groupID, TRUE)) {
479 if ($savedSearchID) {
480 $ssParams = CRM_Contact_BAO_SavedSearch
::getSearchParams($savedSearchID);
482 // rectify params to what proximity search expects if there is a value for prox_distance
484 if (!empty($ssParams)) {
485 CRM_Contact_BAO_ProximityQuery
::fixInputParams($ssParams);
488 if (isset($ssParams['customSearchID'])) {
489 $sql = self
::getCustomSearchSQL($savedSearchID, $ssParams);
492 $sql = self
::getQueryObjectSQL($savedSearchID, $ssParams);
494 $groupID = CRM_Utils_Type
::escape($groupID, 'Integer');
495 $sql['from'] .= " AND contact_a.id NOT IN (
496 SELECT contact_id FROM civicrm_group_contact
497 WHERE civicrm_group_contact.status = 'Removed'
498 AND civicrm_group_contact.group_id = $groupID ) ";
501 if (!empty($sql['select'])) {
502 $sql['select'] = preg_replace("/^\s*SELECT/", "SELECT $groupID as group_id, ", $sql['select']);
505 $groupContactsTempTable = CRM_Utils_SQL_TempTable
::build()->setCategory('gccache')->setMemory();
506 $tempTable = $groupContactsTempTable->getName();
507 $groupContactsTempTable->createWithColumns('contact_id int, group_id int, UNIQUE UI_contact_group (contact_id,group_id)');
509 $contactQueries[] = $sql;
510 // lets also store the records that are explicitly added to the group
511 // this allows us to skip the group contact LEFT JOIN
512 $contactQueries[] = [
513 'select' => "SELECT $groupID as group_id, contact_id as contact_id",
514 'from' => " FROM civicrm_group_contact WHERE civicrm_group_contact.status = 'Added' AND civicrm_group_contact.group_id = $groupID ",
517 self
::clearGroupContactCache($groupID);
519 foreach ($contactQueries as $contactQuery) {
520 if (empty($contactQuery['select']) ||
empty($contactQuery['from'])) {
523 if (CRM_Core_DAO
::singleValueQuery("SELECT COUNT(*) {$contactQuery['from']}") > 0) {
524 CRM_Core_DAO
::executeQuery("INSERT IGNORE INTO $tempTable (group_id, contact_id) {$contactQuery['select']} {$contactQuery['from']}");
528 if ($group->children
) {
530 // Store a list of contacts who are removed from the parent group
531 $sqlContactsRemovedFromGroup = "
533 FROM civicrm_group_contact
534 WHERE civicrm_group_contact.status = 'Removed'
535 AND civicrm_group_contact.group_id = $groupID ";
536 $dao = CRM_Core_DAO
::executeQuery($sqlContactsRemovedFromGroup);
537 $removed_contacts = [];
538 while ($dao->fetch()) {
539 $removed_contacts[] = $dao->contact_id
;
542 $childrenIDs = explode(',', $group->children
);
543 foreach ($childrenIDs as $childID) {
544 $contactIDs = CRM_Contact_BAO_Group
::getMember($childID, FALSE);
545 // Unset each contact that is removed from the parent group
546 foreach ($removed_contacts as $removed_contact) {
547 unset($contactIDs[$removed_contact]);
549 if (empty($contactIDs)) {
550 // This child group has no contact IDs so we don't need to add them to
554 foreach ($contactIDs as $contactID => $dontCare) {
555 $values[] = "({$groupID},{$contactID})";
557 $str = implode(',', $values);
558 CRM_Core_DAO
::executeQuery("INSERT IGNORE INTO $tempTable (group_id, contact_id) VALUES $str");
562 // grab a lock so other processes don't compete and do the same query
563 $lock = Civi
::lockManager()->acquire("data.core.group.{$groupID}");
564 if (!$lock->isAcquired()) {
565 // this can cause inconsistent results since we don't know if the other process
566 // will fill up the cache before our calling routine needs it.
567 // however this routine does not return the status either, so basically
568 // its a "lets return and hope for the best"
572 // Don't call clearGroupContactCache as we don't want to clear the cache dates
573 // The will get updated by updateCacheTime() below and not clearing the dates reduces
574 // the chance that loadAll() will try and rebuild at the same time.
577 FROM civicrm_group_contact_cache g
578 WHERE g.group_id = %1 ";
580 1 => [$groupID, 'Integer'],
582 CRM_Core_DAO
::executeQuery($clearCacheQuery, $params);
584 CRM_Core_DAO
::executeQuery(
585 "INSERT IGNORE INTO civicrm_group_contact_cache (contact_id, group_id)
586 SELECT DISTINCT contact_id, group_id FROM $tempTable
588 $groupContactsTempTable->drop();
589 self
::updateCacheTime([$groupID], TRUE);
595 * Retrieve the smart group cache timeout in minutes.
597 * This checks if a timeout has been configured. If one has then smart groups should not
598 * be refreshed more frequently than the time out. If a group was recently refreshed it should not
599 * refresh again within that period.
603 public static function smartGroupCacheTimeout() {
604 $config = CRM_Core_Config
::singleton();
607 isset($config->smartGroupCacheTimeout
) &&
608 is_numeric($config->smartGroupCacheTimeout
)
610 return $config->smartGroupCacheTimeout
;
613 // Default to 5 minutes.
618 * Get all the smart groups that this contact belongs to.
620 * Note that this could potentially be a super slow function since
621 * it ensure that all contact groups are loaded in the cache
623 * @param int $contactID
624 * @param bool $showHidden
625 * Hidden groups are shown only if this flag is set.
628 * an array of groups that this contact belongs to
630 public static function contactGroup($contactID, $showHidden = FALSE) {
631 if (empty($contactID)) {
635 if (is_array($contactID)) {
636 $contactIDs = $contactID;
639 $contactIDs = [$contactID];
646 $hiddenClause = ' AND (g.is_hidden = 0 OR g.is_hidden IS NULL) ';
649 $contactIDString = CRM_Core_DAO
::escapeString(implode(', ', $contactIDs));
651 SELECT gc.group_id, gc.contact_id, g.title, g.children, g.description
652 FROM civicrm_group_contact_cache gc
653 INNER JOIN civicrm_group g ON g.id = gc.group_id
654 WHERE gc.contact_id IN ($contactIDString)
656 ORDER BY gc.contact_id, g.children
659 $dao = CRM_Core_DAO
::executeQuery($sql);
661 $prevContactID = NULL;
662 while ($dao->fetch()) {
665 $prevContactID != $dao->contact_id
667 $contactGroup[$prevContactID]['groupTitle'] = implode(', ', $contactGroup[$prevContactID]['groupTitle']);
669 $prevContactID = $dao->contact_id
;
670 if (!array_key_exists($dao->contact_id
, $contactGroup)) {
671 $contactGroup[$dao->contact_id
]
672 = ['group' => [], 'groupTitle' => []];
675 $contactGroup[$dao->contact_id
]['group'][]
677 'id' => $dao->group_id
,
678 'title' => $dao->title
,
679 'description' => $dao->description
,
680 'children' => $dao->children
,
682 $contactGroup[$dao->contact_id
]['groupTitle'][] = $dao->title
;
685 if ($prevContactID) {
686 $contactGroup[$prevContactID]['groupTitle'] = implode(', ', $contactGroup[$prevContactID]['groupTitle']);
689 if ((!empty($contactGroup[$contactID]) && is_numeric($contactID))) {
690 return $contactGroup[$contactID];
693 return $contactGroup;
698 * Get the datetime from which the cache should be considered invalid.
700 * Ie if the smartgroup cache timeout is 5 minutes ago then the cache is invalid if it was
701 * refreshed 6 minutes ago, but not if it was refreshed 4 minutes ago.
705 public static function getCacheInvalidDateTime() {
706 return date('YmdHis', strtotime("-" . self
::smartGroupCacheTimeout() . " Minutes"));
710 * Get the date when the cache should be refreshed from.
712 * Ie. now + the offset & we will delete anything prior to then.
716 public static function getRefreshDateTime() {
717 return date('YmdHis', strtotime("+ " . self
::smartGroupCacheTimeout() . " Minutes"));
721 * Invalidates the smart group cache for a particular group
722 * @param int $groupID - Group to invalidate
724 public static function invalidateGroupContactCache($groupID) {
725 CRM_Core_DAO
::executeQuery("UPDATE civicrm_group
726 SET cache_date = NULL, refresh_date = NULL
728 1 => [$groupID, 'Positive'],
733 * Get sql from a custom search.
735 * @param int $savedSearchID
736 * @param array $ssParams
741 protected static function getCustomSearchSQL($savedSearchID, array $ssParams): array {
744 // we split it up and store custom class
745 // so temp tables are not destroyed if they are used
746 // hence customClass is defined above at top of function
747 $customClass = CRM_Contact_BAO_SearchCustom
::customClass($ssParams['customSearchID'], $savedSearchID);
748 $searchSQL = $customClass->contactIDs();
749 $searchSQL = str_replace('ORDER BY contact_a.id ASC', '', $searchSQL);
750 if (strpos($searchSQL, 'WHERE') === FALSE) {
751 $searchSQL .= " WHERE ( 1 ) ";
754 'select' => substr($searchSQL, 0, strpos($searchSQL, 'FROM')),
755 'from' => substr($searchSQL, strpos($searchSQL, 'FROM')),
761 * Get array of sql from a saved query object group.
763 * @param int $savedSearchID
764 * @param array $ssParams
767 * @throws \CRM_Core_Exception
768 * @throws \CiviCRM_API3_Exception
770 protected static function getQueryObjectSQL($savedSearchID, array $ssParams): array {
771 $returnProperties = NULL;
772 if (CRM_Core_DAO
::getFieldValue('CRM_Contact_DAO_SavedSearch', $savedSearchID, 'mapping_id')) {
773 $fv = CRM_Contact_BAO_SavedSearch
::getFormValues($savedSearchID);
774 $returnProperties = CRM_Core_BAO_Mapping
::returnProperties($fv);
776 $formValues = CRM_Contact_BAO_SavedSearch
::getFormValues($savedSearchID);
777 // CRM-17075 using the formValues in this way imposes extra logic and complexity.
778 // we have the where_clause and where tables stored in the saved_search table
779 // and should use these rather than re-processing the form criteria (which over-works
780 // the link between the form layer & the query layer too).
781 // It's hard to think of when you would want to use anything other than return
782 // properties = array('contact_id' => 1) here as the point would appear to be to
783 // generate the list of contact ids in the group.
784 // @todo review this to use values in saved_search table (preferably for 4.8).
786 = new CRM_Contact_BAO_Query(
787 $ssParams, $returnProperties, NULL,
791 CRM_Utils_Array
::value('display_relationship_type', $formValues),
792 CRM_Utils_Array
::value('operator', $formValues, 'AND')
794 $query->_useDistinct
= FALSE;
795 $query->_useGroupBy
= FALSE;
796 $sqlParts = $query->getSearchSQLParts(
802 'select' => $sqlParts['select'],
803 'from' => "{$sqlParts['from']} {$sqlParts['where']} {$sqlParts['having']} {$sqlParts['group_by']}",