From 19a1c852192f9f879bfd8976387356461c057805 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Tue, 1 Feb 2022 21:57:01 -0800 Subject: [PATCH] (Schema) Queue and QueueItem - Track detailed settings (runner, lease_time, retry_count, batch_limit, etc) (5.48) --- CRM/Queue/BAO/QueueItem.php | 37 +++-- CRM/Queue/DAO/Queue.php | 132 ++++++++++++++++-- CRM/Queue/DAO/QueueItem.php | 28 +++- .../Incremental/php/FiveFortyEight.php | 30 ++++ sql/civicrm_queue_item.mysql | 3 + xml/schema/Queue/Queue.xml | 60 +++++++- xml/schema/Queue/QueueItem.xml | 13 +- 7 files changed, 278 insertions(+), 25 deletions(-) diff --git a/CRM/Queue/BAO/QueueItem.php b/CRM/Queue/BAO/QueueItem.php index 3b3f138469..64a415b0da 100644 --- a/CRM/Queue/BAO/QueueItem.php +++ b/CRM/Queue/BAO/QueueItem.php @@ -25,25 +25,46 @@ class CRM_Queue_BAO_QueueItem extends CRM_Queue_DAO_QueueItem { /** * Ensure that the required SQL table exists. * + * The `civicrm_queue_item` table is a special requirement - without it, the upgrader cannot run. + * The upgrader will make a special request for `findCreateTable()` before computing upgrade-tasks. + * * @return bool * TRUE if table now exists */ - public static function findCreateTable() { - $checkTableSql = "show tables like 'civicrm_queue_item'"; - $foundName = CRM_Core_DAO::singleValueQuery($checkTableSql); - if ($foundName == 'civicrm_queue_item') { - return TRUE; + public static function findCreateTable(): bool { + if (!CRM_Core_DAO::checkTableExists('civicrm_queue_item')) { + return static::createTable(); + } + else { + return static::updateTable(); } + } + /** + * Create the `civicrm_queue_item` table. + * + * @return bool + * @throws \CRM_Core_Exception + */ + protected static function createTable(): bool { // civicrm/sql/civicrm_queue_item.mysql $fileName = dirname(__FILE__) . '/../../../sql/civicrm_queue_item.mysql'; $config = CRM_Core_Config::singleton(); CRM_Utils_File::sourceSQLFile($config->dsn, $fileName); - // Make sure it succeeded - $foundName = CRM_Core_DAO::singleValueQuery($checkTableSql); - return ($foundName == 'civicrm_queue_item'); + return CRM_Core_DAO::checkTableExists('civicrm_queue_item'); + } + + /** + * Ensure that the `civicrm_queue_item` table is up-to-date. + * + * @return bool + */ + public static function updateTable(): bool { + CRM_Upgrade_Incremental_Base::addColumn(NULL, 'civicrm_queue_item', 'run_count', + "int NOT NULL DEFAULT 0 COMMENT 'Number of times execution has been attempted.'"); + return TRUE; } } diff --git a/CRM/Queue/DAO/Queue.php b/CRM/Queue/DAO/Queue.php index fc5ce93eff..c3f532711f 100644 --- a/CRM/Queue/DAO/Queue.php +++ b/CRM/Queue/DAO/Queue.php @@ -6,7 +6,7 @@ * * Generated from xml/schema/CRM/Queue/Queue.xml * DO NOT EDIT. Generated by CRM_Core_CodeGen - * (GenCodeChecksum:5a49f8a2765460d48e4266efa9d447f2) + * (GenCodeChecksum:0d2a970bc280fd92df3b3b25ec0d42d0) */ /** @@ -56,13 +56,49 @@ class CRM_Queue_DAO_Queue extends CRM_Core_DAO { public $type; /** - * Should the standard background attempt to autorun tasks in this queue? + * Name of the task runner * - * @var bool|string|null - * (SQL type: tinyint) + * @var string + * (SQL type: varchar(64)) * Note that values will be retrieved from the database as a string. */ - public $is_autorun; + public $runner; + + /** + * Maximum number of items in a batch. Tip: If you expand batch_limit, then also consider expanding lease_time. + * + * @var int|string + * (SQL type: int unsigned) + * Note that values will be retrieved from the database as a string. + */ + public $batch_limit; + + /** + * When claiming an item (or batch of items) for work, how long should the item(s) be reserved. (Seconds) + * + * @var int|string + * (SQL type: int unsigned) + * Note that values will be retrieved from the database as a string. + */ + public $lease_time; + + /** + * Number of permitted retries. Decreases with each retry. Zero (0) to disable. Null for system default. + * + * @var int|string + * (SQL type: int) + * Note that values will be retrieved from the database as a string. + */ + public $retry_limit; + + /** + * Number of seconds to wait before retrying a failed execution. + * + * @var int|string + * (SQL type: int) + * Note that values will be retrieved from the database as a string. + */ + public $retry_interval; /** * Class constructor. @@ -145,21 +181,89 @@ class CRM_Queue_DAO_Queue extends CRM_Core_DAO { ], 'add' => '5.47', ], - 'is_autorun' => [ - 'name' => 'is_autorun', - 'type' => CRM_Utils_Type::T_BOOLEAN, - 'title' => ts('Enable Autorun'), - 'description' => ts('Should the standard background attempt to autorun tasks in this queue?'), - 'where' => 'civicrm_queue.is_autorun', + 'runner' => [ + 'name' => 'runner', + 'type' => CRM_Utils_Type::T_STRING, + 'title' => ts('Runner'), + 'description' => ts('Name of the task runner'), + 'required' => FALSE, + 'maxlength' => 64, + 'size' => CRM_Utils_Type::BIG, + 'where' => 'civicrm_queue.runner', + 'table_name' => 'civicrm_queue', + 'entity' => 'Queue', + 'bao' => 'CRM_Queue_BAO_Queue', + 'localizable' => 0, + 'html' => [ + 'type' => 'Text', + ], + 'add' => '5.48', + ], + 'batch_limit' => [ + 'name' => 'batch_limit', + 'type' => CRM_Utils_Type::T_INT, + 'title' => ts('Batch Limit'), + 'description' => ts('Maximum number of items in a batch. Tip: If you expand batch_limit, then also consider expanding lease_time.'), + 'required' => TRUE, + 'where' => 'civicrm_queue.batch_limit', + 'default' => '1', 'table_name' => 'civicrm_queue', 'entity' => 'Queue', 'bao' => 'CRM_Queue_BAO_Queue', 'localizable' => 0, 'html' => [ - 'type' => 'CheckBox', - 'label' => ts("Auto Run"), + 'type' => 'Text', + ], + 'add' => '5.48', + ], + 'lease_time' => [ + 'name' => 'lease_time', + 'type' => CRM_Utils_Type::T_INT, + 'title' => ts('Lease Time'), + 'description' => ts('When claiming an item (or batch of items) for work, how long should the item(s) be reserved. (Seconds)'), + 'required' => TRUE, + 'where' => 'civicrm_queue.lease_time', + 'default' => '3600', + 'table_name' => 'civicrm_queue', + 'entity' => 'Queue', + 'bao' => 'CRM_Queue_BAO_Queue', + 'localizable' => 0, + 'html' => [ + 'type' => 'Text', + ], + 'add' => '5.48', + ], + 'retry_limit' => [ + 'name' => 'retry_limit', + 'type' => CRM_Utils_Type::T_INT, + 'title' => ts('Retry Limit'), + 'description' => ts('Number of permitted retries. Decreases with each retry. Zero (0) to disable. Null for system default.'), + 'required' => FALSE, + 'where' => 'civicrm_queue.retry_limit', + 'table_name' => 'civicrm_queue', + 'entity' => 'Queue', + 'bao' => 'CRM_Queue_BAO_Queue', + 'localizable' => 0, + 'html' => [ + 'type' => 'Text', + ], + 'add' => '5.48', + ], + 'retry_interval' => [ + 'name' => 'retry_interval', + 'type' => CRM_Utils_Type::T_INT, + 'title' => ts('Retry Interval'), + 'description' => ts('Number of seconds to wait before retrying a failed execution.'), + 'required' => FALSE, + 'where' => 'civicrm_queue.retry_interval', + 'table_name' => 'civicrm_queue', + 'entity' => 'Queue', + 'bao' => 'CRM_Queue_BAO_Queue', + 'localizable' => 0, + 'html' => [ + 'type' => 'Text', ], - 'add' => NULL, + 'add' => '5.48', ], ]; CRM_Core_DAO_AllCoreTables::invoke(__CLASS__, 'fields_callback', Civi::$statics[__CLASS__]['fields']); diff --git a/CRM/Queue/DAO/QueueItem.php b/CRM/Queue/DAO/QueueItem.php index 7d4979ca95..0ad5fe3be6 100644 --- a/CRM/Queue/DAO/QueueItem.php +++ b/CRM/Queue/DAO/QueueItem.php @@ -6,7 +6,7 @@ * * Generated from xml/schema/CRM/Queue/QueueItem.xml * DO NOT EDIT. Generated by CRM_Core_CodeGen - * (GenCodeChecksum:36871610524adb64bc8aa3bb24b295f4) + * (GenCodeChecksum:f5163d86b425127deb25d105976212bf) */ /** @@ -71,6 +71,15 @@ class CRM_Queue_DAO_QueueItem extends CRM_Core_DAO { */ public $release_time; + /** + * Number of times execution has been attempted. + * + * @var int|string + * (SQL type: int) + * Note that values will be retrieved from the database as a string. + */ + public $run_count; + /** * Serialized queue data * @@ -188,6 +197,23 @@ class CRM_Queue_DAO_QueueItem extends CRM_Core_DAO { ], 'add' => NULL, ], + 'run_count' => [ + 'name' => 'run_count', + 'type' => CRM_Utils_Type::T_INT, + 'title' => ts('Run Count'), + 'description' => ts('Number of times execution has been attempted.'), + 'required' => TRUE, + 'where' => 'civicrm_queue_item.run_count', + 'default' => '0', + 'table_name' => 'civicrm_queue_item', + 'entity' => 'QueueItem', + 'bao' => 'CRM_Queue_BAO_QueueItem', + 'localizable' => 0, + 'html' => [ + 'type' => 'Text', + ], + 'add' => '5.48', + ], 'data' => [ 'name' => 'data', 'type' => CRM_Utils_Type::T_LONGTEXT, diff --git a/CRM/Upgrade/Incremental/php/FiveFortyEight.php b/CRM/Upgrade/Incremental/php/FiveFortyEight.php index 2f8b3bfdb2..3cadf4fea3 100644 --- a/CRM/Upgrade/Incremental/php/FiveFortyEight.php +++ b/CRM/Upgrade/Incremental/php/FiveFortyEight.php @@ -29,6 +29,36 @@ class CRM_Upgrade_Incremental_php_FiveFortyEight extends CRM_Upgrade_Incremental */ public function upgrade_5_48_alpha1($rev): void { $this->addTask(ts('Upgrade DB to %1: SQL', [1 => $rev]), 'runSql', $rev); + $this->addTask('Add "runner" to "civicrm_queue"', 'addColumn', 'civicrm_queue', 'runner', + "varchar(64) NULL COMMENT 'Name of the task runner'" + ); + $this->addTask('Convert "is_autorun" to "runner"', 'convertAutorun'); + $this->addTask('Drop "is_autorun" from "civicrm_queue"', 'dropColumn', 'civicrm_queue', 'is_autorun'); + $this->addTask('Add "batch_limit" to "civicrm_queue"', 'addColumn', 'civicrm_queue', 'batch_limit', + "int unsigned NOT NULL DEFAULT 1 COMMENT 'Maximum number of items in a batch.'" + ); + $this->addTask('Add "lease_time" to "civicrm_queue"', 'addColumn', 'civicrm_queue', 'lease_time', + "int unsigned NOT NULL DEFAULT 3600 COMMENT 'When claiming an item (or batch of items) for work, how long should the item(s) be reserved. (Seconds)'" + ); + $this->addTask('Add "retry_limit" to "civicrm_queue"', 'addColumn', 'civicrm_queue', 'retry_limit', + "int NULL COMMENT 'Number of permitted retries. Decreases with each retry. Zero (0) to disable. Null for system default.'" + ); + $this->addTask('Add "retry_interval" to "civicrm_queue"', 'addColumn', 'civicrm_queue', 'retry_interval', + "int NULL COMMENT 'Number of seconds to wait before retrying a failed execution.'" + ); + } + + /** + * The `is_autorun` column was introduced in 5.47, but we didn't finish adding the + * additional changes to use, so there shouldn't be any real usage. But just to be + * paranoid, we'll convert to 5.48's `runner`. + * + * @param \CRM_Queue_TaskContext $ctx + * @return bool + */ + public static function convertAutorun(CRM_Queue_TaskContext $ctx) { + CRM_Core_DAO::executeQuery('UPDATE civicrm_queue SET runner = "task" WHERE is_autorun = 1'); + return TRUE; } } diff --git a/sql/civicrm_queue_item.mysql b/sql/civicrm_queue_item.mysql index 8f27343a84..9ccffb56f4 100644 --- a/sql/civicrm_queue_item.mysql +++ b/sql/civicrm_queue_item.mysql @@ -16,6 +16,9 @@ CREATE TABLE `civicrm_queue_item` ( `weight` int NOT NULL , `submit_time` datetime NOT NULL COMMENT 'date on which this item was submitted to the queue', `release_time` datetime COMMENT 'date on which this job becomes available; null if ASAP', + `retry_interval` int NULL COMMENT 'Number of seconds to wait before retrying a failed execution. NULL to disable.', + `retry_count` int NULL COMMENT 'Number of permitted retries. Decreases with each retry. NULL to disable.', + `data` text COMMENT 'Serialized queue' , PRIMARY KEY ( `id` ) diff --git a/xml/schema/Queue/Queue.xml b/xml/schema/Queue/Queue.xml index 176f70b0e3..53752e9550 100644 --- a/xml/schema/Queue/Queue.xml +++ b/xml/schema/Queue/Queue.xml @@ -66,6 +66,64 @@ CheckBox + 5.48 + + + runner + Runner + varchar + 64 + false + Name of the task runner + + Text + + 5.48 + + + batch_limit + Batch Limit + int unsigned + true + 1 + Maximum number of items in a batch. Tip: If you expand batch_limit, then also consider expanding lease_time. + 5.48 + + Text + + + + lease_time + Lease Time + int unsigned + true + 3600 + When claiming an item (or batch of items) for work, how long should the item(s) be reserved. (Seconds) + 5.48 + + Text + + + + retry_limit + Retry Limit + Number of permitted retries. Decreases with each retry. Zero (0) to disable. Null for system default. + int + false + + Text + + 5.48 + + + retry_interval + Retry Interval + Number of seconds to wait before retrying a failed execution. + int + false + + Text + + 5.48 - diff --git a/xml/schema/Queue/QueueItem.xml b/xml/schema/Queue/QueueItem.xml index 1419b0ab84..3d3bea186b 100644 --- a/xml/schema/Queue/QueueItem.xml +++ b/xml/schema/Queue/QueueItem.xml @@ -65,7 +65,18 @@ activityDateTime - + + run_count + Run Count + Number of times execution has been attempted. + int + 0 + true + + Text + + 5.48 + data Queue item data -- 2.25.1