static $_alreadyLoaded = array();
+ /**
+ * Get a list of caching modes.
+ *
+ * @return array
+ */
+ public static function getModes() {
+ return array(
+ // Flush expired caches in response to user actions.
+ 'opportunistic' => ts('Opportunistic Flush'),
+
+ // Flush expired caches via background cron jobs.
+ 'deterministic' => ts('Cron Flush'),
+ );
+ }
+
/**
* Check to see if we have cache entries for this group.
*
* If not, regenerate, else return.
*
- * @param $groupIDs
+ * @param array $groupIDs
* Of group that we are checking against.
*
* @return bool
}
/**
- * Common function that formulates the query to see which groups needs to be refreshed
- * based on their cache date and the smartGroupCacheTimeOut
+ * Formulate the query to see which groups needs to be refreshed.
+ *
+ * The calculation is based on their cache date and the smartGroupCacheTimeOut
*
* @param string $groupIDClause
* The clause which limits which groups we need to evaluate.
* the sql query which lists the groups that need to be refreshed
*/
public static function groupRefreshedClause($groupIDClause = NULL, $includeHiddenGroups = FALSE) {
- $smartGroupCacheTimeout = self::smartGroupCacheTimeout();
- $now = CRM_Utils_Date::getUTCTime();
+ $smartGroupCacheTimeoutDateTime = self::getCacheInvalidDateTime();
$query = "
SELECT g.id
FROM civicrm_group g
WHERE ( g.saved_search_id IS NOT NULL OR g.children IS NOT NULL )
AND g.is_active = 1
-AND ( g.cache_date IS NULL OR
- ( TIMESTAMPDIFF(MINUTE, g.cache_date, $now) >= $smartGroupCacheTimeout ) OR
- ( $now >= g.refresh_date )
- )
-";
+AND (
+ g.cache_date IS NULL
+ OR cache_date <= $smartGroupCacheTimeoutDateTime
+ OR NOW() >= g.refresh_date
+)";
if (!$includeHiddenGroups) {
$query .= "AND (g.is_hidden = 0 OR g.is_hidden IS NULL)";
}
/**
- * Checks to see if a group has been refreshed recently. This is primarily used
- * in a locking scenario when some other process might have refreshed things underneath
+ * Check to see if a group has been refreshed recently.
+ *
+ * This is primarily used in a locking scenario when some other process might have refreshed things underneath
* this process
*
* @param int $groupID
}
/**
- * Check to see if we have cache entries for this group
+ * Check to see if we have cache entries for this group.
+ *
* if not, regenerate, else return
*
- * @param int /array $groupIDs groupIDs of group that we are checking against
+ * @param int|array $groupIDs groupIDs of group that we are checking against
* if empty, all groups are checked
* @param int $limit
* Limits the number of groups we evaluate.
if (!empty($refreshGroupIDs)) {
$refreshGroupIDString = CRM_Core_DAO::escapeString(implode(', ', $refreshGroupIDs));
- $time = CRM_Utils_Date::getUTCTime(self::smartGroupCacheTimeout() * 60);
+ $time = self::getRefreshDateTime();
$query = "
UPDATE civicrm_group g
SET g.refresh_date = $time
}
/**
- * FIXME: This function should not be needed, because the cache table should not be getting truncated
- */
- public static function fillIfEmpty() {
- if (!CRM_Core_DAO::singleValueQuery("SELECT COUNT(id) FROM civicrm_group_contact_cache")) {
- self::loadAll();
- }
- }
-
- /**
+ * Build the smart group cache for a given group.
+ *
* @param int $groupID
*/
public static function add($groupID) {
}
/**
+ * Store values into the group contact cache.
+ *
+ * @todo review use of INSERT IGNORE. This function appears to be slower that inserting
+ * with a left join. Also, 200 at once seems too little.
+ *
* @param int $groupID
- * @param $values
+ * @param array $values
*/
public static function store(&$groupID, &$values) {
$processed = FALSE;
// only update cache entry if we had any values
if ($processed) {
// also update the group with cache date information
- //make sure to give original timezone settings again.
- $now = CRM_Utils_Date::getUTCTime();
+ $now = date('YmdHis');
$refresh = 'null';
}
else {
/**
* Removes all the cache entries pertaining to a specific group.
+ *
* If no groupID is passed in, removes cache entries for all groups
* Has an optimization to bypass repeated invocations of this function.
* Note that this function is an advisory, i.e. the removal respects the
* cache date, i.e. the removal is not done if the group was recently
* loaded into the cache.
*
+ * In fact it turned out there is little overlap between the code when group is passed in
+ * and group is not so it makes more sense as separate functions.
+ *
+ * @todo remove last call to this function from outside the class then make function protected,
+ * enforce groupID as an array & remove non group handling.
+ *
* @param int $groupID
* the groupID to delete cache entries, NULL for all groups.
* @param bool $onceOnly
static $invoked = FALSE;
// typically this needs to happy only once per instance
- // this is especially TRUE in import, where we dont need
+ // this is especially TRUE in import, where we don't need
// to do this all the time
// this optimization is done only when no groupID is passed
// i.e. cache is reset for all groups
}
$refresh = NULL;
- $params = array();
$smartGroupCacheTimeout = self::smartGroupCacheTimeout();
-
- $now = CRM_Utils_Date::getUTCTime();
- $refreshTime = CRM_Utils_Date::getUTCTime($smartGroupCacheTimeout * 60);
+ $params = array(
+ 1 => array(self::getCacheInvalidDateTime(), 'String'),
+ 2 => array(self::getRefreshDateTime(), 'String'),
+ );
if (!isset($groupID)) {
if ($smartGroupCacheTimeout == 0) {
$query = "
-TRUNCATE civicrm_group_contact_cache
+DELETE FROM civicrm_group_contact_cache
";
$update = "
UPDATE civicrm_group g
";
$refresh = "
UPDATE civicrm_group g
-SET refresh_date = $refreshTime
-WHERE g.cache_date > %1
+SET refresh_date = %2
+WHERE g.cache_date < %1
AND refresh_date IS NULL
";
- $cacheTime = date('Y-m-d H-i-s', strtotime("- $smartGroupCacheTimeout minutes"));
- $params = array(1 => array($cacheTime, 'String'));
}
}
elseif (is_array($groupID)) {
}
/**
- * Removes one or more contacts from the smart group cache.
+ * Refresh the smart group cache tables.
+ *
+ * This involves clearing out any aged entries (based on the site timeout setting) and resetting the time outs.
+ *
+ * This function should be called via the opportunistic or deterministic cache refresh function to make the intent
+ * clear.
+ */
+ protected static function flushCaches() {
+ try {
+ $lock = self::getLockForRefresh();
+ }
+ catch (CRM_Core_Exception $e) {
+ // Someone else is kindly doing the refresh for us right now.
+ return;
+ }
+ $params = array(1 => array(self::getCacheInvalidDateTime(), 'String'));
+ // @todo this is consistent with previous behaviour but as the first query could take several seconds the second
+ // could become inaccurate. It seems to make more sense to fetch them first & delete from an array (which would
+ // also reduce joins). If we do this we should also consider how best to iterate the groups. If we do them one at
+ // a time we could call a hook, allowing people to manage the frequency on their groups, or possibly custom searches
+ // might do that too. However, for 2000 groups that's 2000 iterations. If we do all once we potentially create a
+ // slow query. It's worth noting the speed issue generally relates to the size of the group but if one slow group
+ // is in a query with 500 fast ones all 500 get locked. One approach might be to calculate group size or the
+ // number of groups & then process all at once or many query runs depending on what is found. Of course those
+ // preliminary queries would need speed testing.
+ CRM_Core_DAO::executeQuery(
+ "
+ DELETE gc
+ FROM civicrm_group_contact_cache gc
+ INNER JOIN civicrm_group g ON g.id = gc.group_id
+ WHERE g.cache_date <= %1
+ ",
+ $params
+ );
+
+ // Clear these out without resetting them because we are not building caches here, only clearing them,
+ // so the state is 'as if they had never been built'.
+ CRM_Core_DAO::executeQuery(
+ "
+ UPDATE civicrm_group g
+ SET cache_date = NULL,
+ refresh_date = NULL
+ WHERE g.cache_date <= %1
+ ",
+ $params
+ );
+ $lock->release();
+ }
+
+ /**
+ * Check if the refresh is already initiated.
+ *
+ * We have 2 imperfect methods for this:
+ * 1) a static variable in the function. This works fine within a request
+ * 2) a mysql lock. This works fine as long as CiviMail is not running, or if mysql is version 5.7+
+ *
+ * Where these 2 locks fail we get 2 processes running at the same time, but we have at least minimised that.
+ *
+ * @return \Civi\Core\Lock\LockInterface
+ * @throws \CRM_Core_Exception
+ */
+ protected static function getLockForRefresh() {
+ if (!isset(Civi::$statics[__CLASS__])) {
+ Civi::$statics[__CLASS__] = array('is_refresh_init' => FALSE);
+ }
+
+ if (Civi::$statics[__CLASS__]['is_refresh_init']) {
+ throw new CRM_Core_Exception('A refresh has already run in this process');
+ }
+ $lock = Civi::lockManager()->acquire('data.core.group.refresh');
+ if ($lock->isAcquired()) {
+ Civi::$statics[__CLASS__]['is_refresh_init'] = TRUE;
+ return $lock;
+ }
+ throw new CRM_Core_Exception('Mysql lock unavailable');
+ }
+
+ /**
+ * Do an opportunistic cache refresh if the site is configured for these.
+ *
+ * Sites that do not run the smart group clearing cron job should refresh the caches under an opportunistic mode, akin
+ * to a poor man's cron. The user session will be forced to wait on this so it is less desirable.
+ */
+ public static function opportunisticCacheFlush() {
+ if (Civi::settings()->get('smart_group_cache_refresh_mode') == 'opportunistic') {
+ self::flushCaches();
+ }
+ }
+
+ /**
+ * Do a forced cache refresh.
+ *
+ * This function is appropriate to be called by system jobs & non-user sessions.
+ */
+ public static function deterministicCacheFlush() {
+ if (self::smartGroupCacheTimeout() == 0) {
+ CRM_Core_DAO::executeQuery("TRUNCATE civicrm_group_contact_cache");
+ CRM_Core_DAO::executeQuery("
+ UPDATE civicrm_group g
+ SET cache_date = null, refresh_date = null");
+ }
+ else {
+ self::flushCaches();
+ }
+ }
+
+ /**
+ * Remove one or more contacts from the smart group cache.
+ *
* @param int|array $cid
* @param int $groupId
+ *
* @return bool
* TRUE if successful.
*/
return;
}
- // grab a lock so other processes dont compete and do the same query
+ // grab a lock so other processes don't compete and do the same query
$lock = Civi::lockManager()->acquire("data.core.group.{$groupID}");
if (!$lock->isAcquired()) {
- // this can cause inconsistent results since we dont know if the other process
+ // this can cause inconsistent results since we don't know if the other process
// will fill up the cache before our calling routine needs it.
// however this routine does not return the status either, so basically
// its a "lets return and hope for the best"
}
$insertSql = "CREATE TEMPORARY TABLE $tempTable ($selectSql);";
$processed = TRUE;
- $result = CRM_Core_DAO::executeQuery($insertSql);
+ CRM_Core_DAO::executeQuery($insertSql);
CRM_Core_DAO::executeQuery(
"INSERT IGNORE INTO civicrm_group_contact_cache (contact_id, group_id)
SELECT DISTINCT $idName, group_id FROM $tempTable
}
/**
+ * Retrieve the smart group cache timeout in minutes.
+ *
+ * This checks if a timeout has been configured. If one has then smart groups should not
+ * be refreshed more frequently than the time out. If a group was recently refreshed it should not
+ * refresh again within that period.
+ *
* @return int
*/
public static function smartGroupCacheTimeout() {
return $config->smartGroupCacheTimeout;
}
- // lets have a min cache time of 5 mins if not set
+ // Default to 5 minutes.
return 5;
}
/**
* Get all the smart groups that this contact belongs to.
+ *
* Note that this could potentially be a super slow function since
* it ensure that all contact groups are loaded in the cache
*
}
}
+ /**
+ * Get the datetime from which the cache should be considered invalid.
+ *
+ * Ie if the smartgroup cache timeout is 5 minutes ago then the cache is invalid if it was
+ * refreshed 6 minutes ago, but not if it was refreshed 4 minutes ago.
+ *
+ * @return string
+ */
+ public static function getCacheInvalidDateTime() {
+ return date('YmdHis', strtotime("-" . self::smartGroupCacheTimeout() . " Minutes"));
+ }
+
+ /**
+ * Get the date when the cache should be refreshed from.
+ *
+ * Ie. now + the offset & we will delete anything prior to then.
+ *
+ * @return string
+ */
+ public static function getRefreshDateTime() {
+ return date('YmdHis', strtotime("+ " . self::smartGroupCacheTimeout() . " Minutes"));
+ }
+
}
projects / civicrm-core.git / blobdiff