From 801bafd7ea08b468d8f34efb7f3dfe230694d304 Mon Sep 17 00:00:00 2001 From: eileenmcnaugton Date: Mon, 16 May 2016 13:25:03 +1200 Subject: [PATCH] CRM-16642 add setting & functions for opportunistic vs deterministic cache clearing --- CRM/Contact/BAO/GroupContactCache.php | 91 ++++++++++++++++++++++- CRM/Core/BAO/OptionGroup.php | 26 +++++++ CRM/Upgrade/Incremental/php/FourSeven.php | 35 +++++++++ api/v3/OptionGroup.php | 1 + settings/Core.setting.php | 17 +++++ xml/templates/civicrm_data.tpl | 8 +- 6 files changed, 175 insertions(+), 3 deletions(-) diff --git a/CRM/Contact/BAO/GroupContactCache.php b/CRM/Contact/BAO/GroupContactCache.php index f74912c432..8cbea8e0e8 100644 --- a/CRM/Contact/BAO/GroupContactCache.php +++ b/CRM/Contact/BAO/GroupContactCache.php @@ -402,6 +402,95 @@ WHERE id = %1 CRM_Core_DAO::executeQuery($update, $params); } + /** + * Refresh the smart group cache tables. + * + * This involves clearing out any aged entries (based on the site timeout setting) and resetting the time outs. + * + * This function should be called via the opportunistic or deterministic cache refresh function to make the intent + * clear. + */ + protected static function refreshCaches() { + if (self::isRefreshAlreadyInitiated()) { + return; + } + + $params = array(1 => array(self::getCacheInvalidDateTime(), 'String')); + + CRM_Core_DAO::executeQuery( + " + DELETE gc + FROM civicrm_group_contact_cache gc + INNER JOIN civicrm_group g ON g.id = gc.group_id + WHERE g.cache_date <= %1 + ", + $params + ); + + CRM_Core_DAO::executeQuery( + " + UPDATE civicrm_group g + SET cache_date = null, + refresh_date = NOW() + WHERE g.cache_date <= %1 + ", + $params + ); + } + + /** + * Check if the refresh is already initiated. + * + * We have 2 imperfect methods for this: + * 1) a static variable in the function. This works fine within a request + * 2) a mysql lock. This works fine as long as CiviMail is not running, or if mysql is version 5.7+ + * + * Where these 2 locks fail we get 2 processes running at the same time, but we have at least minimised that. + */ + protected static function isRefreshAlreadyInitiated() { + static $invoked = FALSE; + if ($invoked) { + return TRUE; + } + $lock = Civi::lockManager()->acquire('data.core.group.refresh'); + if (!$lock->isAcquired()) { + return TRUE; + } + } + + /** + * Do an opportunistic cache refresh if the site is configured for these. + * + * Sites that do not run the smart group clearing cron job should refresh the caches under an opportunistic mode, akin + * to a poor man's cron. The user session will be forced to wait on this so it is less desirable. + * + * @return bool + */ + public static function opportunisticCacheRefresh() { + if (Civi::settings()->get('contact_smart_group_display') == 'opportunistic') { + self::refreshCaches(); + } + } + + /** + * Do a forced cache refresh. + * + * This function is appropriate to be called by system jobs & non-user sessions. + * + * @return bool + */ + public static function deterministicCacheRefresh() { + if (self::smartGroupCacheTimeout() == 0) { + CRM_Core_DAO::executeQuery("TRUNCATE civicrm_group_contact_cache"); + CRM_Core_DAO::executeQuery(" + UPDATE civicrm_group g + SET cache_date = null, refresh_date = null"); + } + else { + self::refreshCaches(); + } + } + /** * Remove one or more contacts from the smart group cache. * @@ -558,7 +647,7 @@ WHERE civicrm_group_contact.status = 'Added' } $insertSql = "CREATE TEMPORARY TABLE $tempTable ($selectSql);"; $processed = TRUE; - $result = CRM_Core_DAO::executeQuery($insertSql); + CRM_Core_DAO::executeQuery($insertSql); CRM_Core_DAO::executeQuery( "INSERT IGNORE INTO civicrm_group_contact_cache (contact_id, group_id) SELECT DISTINCT $idName, group_id FROM $tempTable diff --git a/CRM/Core/BAO/OptionGroup.php b/CRM/Core/BAO/OptionGroup.php index 9a72f90042..b9758cbd05 100644 --- a/CRM/Core/BAO/OptionGroup.php +++ b/CRM/Core/BAO/OptionGroup.php @@ -141,4 +141,30 @@ class CRM_Core_BAO_OptionGroup extends CRM_Core_DAO_OptionGroup { return $optionGroup->name; } + /** + * Ensure an option group exists. + * + * This function is intended to be called from the upgrade script to ensure + * that an option group exists, without hitting an error if it already exists. + * + * This is sympathetic to sites who might pre-add it. + * + * @param array $params + * + * @return int + * ID of the option group. + */ + public static function ensureOptionGroupExists($params) { + $existingValues = civicrm_api3('OptionGroup', 'get', array( + 'name' => $params['name'], + )); + if (!$existingValues['count']) { + $result = civicrm_api3('OptionGroup', 'create', $params); + return $result['id']; + } + else { + return $existingValues['id']; + } + } + } diff --git a/CRM/Upgrade/Incremental/php/FourSeven.php b/CRM/Upgrade/Incremental/php/FourSeven.php index 6a33eceda4..c4be97e4a3 100644 --- a/CRM/Upgrade/Incremental/php/FourSeven.php +++ b/CRM/Upgrade/Incremental/php/FourSeven.php @@ -208,6 +208,7 @@ class CRM_Upgrade_Incremental_php_FourSeven extends CRM_Upgrade_Incremental_Base public function upgrade_4_7_8($rev) { $this->addTask(ts('Upgrade DB to %1: SQL', array(1 => $rev)), 'runSql', $rev); $this->addTask('Upgrade mailing foreign key constraints', 'upgradeMailingFKs'); + $this->addSmartGroupRefreshOptions(); } /* @@ -690,4 +691,38 @@ FROM `civicrm_dashboard_contact` JOIN `civicrm_contact` WHERE civicrm_dashboard_ return TRUE; } + /** + * CRM-16642 Add option for smart group refreshing. + * + * @param \CRM_Queue_TaskContext $ctx + * + * @return bool + */ + public function addSmartGroupRefreshOptions(CRM_Queue_TaskContext $ctx) { + $optionGroupID = CRM_Core_BAO_OptionGroup::ensureOptionGroupExists(array( + 'name' => 'smart_group_cache_refresh_mode', + 'title' => ts('Mode for refreshing smart group cache'), + 'description' => ts('This provides the option for the smart group cache setting'), + 'is_reserved' => 1, + )); + CRM_Core_BAO_OptionValue::ensureOptionValueExists(array( + 'option_group_id' => $optionGroupID, + 'name' => 'opportunistic', + 'label' => ts('Opportunistic'), + 'description' => ts('Purge the cache in response to user actions'), + 'is_active' => TRUE, + 'filter' => 1, + 'is_reserved' => 1, + )); + CRM_Core_BAO_OptionValue::ensureOptionValueExists(array( + 'option_group_id' => $optionGroupID, + 'name' => 'deterministic', + 'label' => ts('Deterministic'), + 'description' => ts('Only purge the cache on system jobs'), + 'is_active' => TRUE, + 'filter' => 1, + 'is_reserved' => 1, + )); + } + } diff --git a/api/v3/OptionGroup.php b/api/v3/OptionGroup.php index c40bc3fd38..c90bb6d85d 100644 --- a/api/v3/OptionGroup.php +++ b/api/v3/OptionGroup.php @@ -66,6 +66,7 @@ function civicrm_api3_option_group_create($params) { */ function _civicrm_api3_option_group_create_spec(&$params) { $params['name']['api.unique'] = 1; + $params['is_active']['api.default'] = TRUE; } /** diff --git a/settings/Core.setting.php b/settings/Core.setting.php index ce2c88fa9b..daadbe0e67 100644 --- a/settings/Core.setting.php +++ b/settings/Core.setting.php @@ -294,6 +294,23 @@ return array( 'description' => NULL, 'help_text' => NULL, ), + 'smart_group_cache_refresh_mode' => array( + 'group_name' => 'CiviCRM Preferences', + 'group' => 'core', + 'name' => 'smart_group_cache_refresh_mode', + 'type' => 'String', + 'html_type' => 'radio', + 'default' => 'opportunistic', + 'add' => '4.7', + 'title' => 'Smart Group Refresh Mode', + 'is_domain' => 1, + 'is_contact' => 0, + 'pseudoconstant' => array( + 'optionGroupName' => 'smart_group_cache_refresh_mode', + ), + 'description' => 'Should the smart groups be by cron jobs or user actions', + 'help_text' => 'If you are not in a position to configure the cron you should leave this at the default. If you are then clearing caches via cron will improve the user experience.', + ), 'installed' => array( 'bootstrap_comment' => 'This is a boot setting which may be loaded during bootstrap. Defaults are loaded via SettingsBag::getSystemDefaults().', 'group_name' => 'CiviCRM Preferences', diff --git a/xml/templates/civicrm_data.tpl b/xml/templates/civicrm_data.tpl index eff7f199e9..436d3dc401 100644 --- a/xml/templates/civicrm_data.tpl +++ b/xml/templates/civicrm_data.tpl @@ -209,7 +209,8 @@ VALUES ('communication_style' , '{ts escape="sql"}Communication Style{/ts}' , 1, 1, 0), ('msg_mode' , '{ts escape="sql"}Message Mode{/ts}' , 1, 1, 0), ('contact_date_reminder_options' , '{ts escape="sql"}Contact Date Reminder Options{/ts}' , 1, 1, 1), - ('relative_date_filters' , '{ts escape="sql"}Relative Date Filters{/ts}' , 1, 1, 0); + ('relative_date_filters' , '{ts escape="sql"}Relative Date Filters{/ts}' , 1, 1, 0), + ('smart_group_cache_refresh_mode', '{ts escape="sql"}Smart Group Cache Management Mode{/ts}' , 1, 1, 1); SELECT @option_group_id_pcm := max(id) from civicrm_option_group where name = 'preferred_communication_method'; SELECT @option_group_id_act := max(id) from civicrm_option_group where name = 'activity_type'; @@ -288,6 +289,7 @@ SELECT @option_group_id_communication_style := max(id) from civicrm_option_group SELECT @option_group_id_msg_mode := max(id) from civicrm_option_group where name = 'msg_mode'; SELECT @option_group_id_contactDateMode := max(id) from civicrm_option_group where name = 'contact_date_reminder_options'; SELECT @option_group_id_date_filter := max(id) from civicrm_option_group where name = 'relative_date_filters'; +SELECT @option_group_smart_group_cache_refresh_mode := max(id) from civicrm_option_group where name = 'smart_group_cache_refresh_mode'; SELECT @contributeCompId := max(id) FROM civicrm_component where name = 'CiviContribute'; SELECT @eventCompId := max(id) FROM civicrm_component where name = 'CiviEvent'; @@ -1018,7 +1020,9 @@ VALUES (@option_group_id_date_filter, '{ts escape="sql"}From end of previous week{/ts}', 'greater_previous.week', 'greater_previous.week', NULL, NULL, NULL,59, NULL, 0, 0, 1, NULL, NULL), (@option_group_id_date_filter, '{ts escape="sql"}From end of previous calendar month{/ts}', 'greater_previous.month', 'greater_previous.month', NULL, NULL, NULL,60, NULL, 0, 0, 1, NULL, NULL), (@option_group_id_date_filter, '{ts escape="sql"}From end of previous quarter{/ts}', 'greater_previous.quarter', 'greater_previous.quarter', NULL, NULL, NULL,61, NULL, 0, 0, 1, NULL, NULL), - (@option_group_id_date_filter, '{ts escape="sql"}From end of previous calendar year{/ts}', 'greater_previous.year', 'greater_previous.year', NULL, NULL, NULL,62, NULL, 0, 0, 1, NULL, NULL); + (@option_group_id_date_filter, '{ts escape="sql"}From end of previous calendar year{/ts}', 'greater_previous.year', 'greater_previous.year', NULL, NULL, NULL,62, NULL, 0, 0, 1, NULL, NULL), + (@option_group_smart_group_cache_refresh_mode, '{ts escape="sql"}Opportunistic{/ts}', 'opportunistic','opportunistic', NULL, NULL, NULL,1, NULL, 0, 0, 1, NULL, NULL), + (@option_group_smart_group_cache_refresh_mode, '{ts escape="sql"}Deterministic{/ts}', 'deterministic','deterministic', NULL, NULL, NULL,1, NULL, 0, 0, 2, NULL, NULL); -- financial accounts SELECT @opval := value FROM civicrm_option_value WHERE name = 'Revenue' and option_group_id = @option_group_id_fat; -- 2.25.1