From 92ee7b1934727848a2c5e2debf3a41b553f81d2b Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Tue, 14 Jul 2020 00:29:07 -0700 Subject: [PATCH] civix#175 - Add support for mixins. Use MixinScanner/MixinLoader and boot-cache. Overview -------- (NOTE: For this description, I reference the term "API" in the general sense of a programmatic interface -- such as a hook or file-naming convention. It is not specifically about CRUD/DB APIs.) The `civix` code-generator provides support for additional coding-conventions -- ones which are more amenable to code-generation. For example, it autoloads files from `xml/Menu/*.xml` and `**/*.mgd.php`. The technique for implementing this traditionally relies on generating a lot of boilerplate. This patch introduces a new construct ("mixin") which allows boilerplate to be maintained more easily. A mixin inspects an extension programmatically, registering new hooks as needed. A mixin may start out as a file in `civix` (or even as a bespoke file in some module) - and then be migrated into `civicrm-core`. Each mixin has a name and version, which means that (at runtime) it will only load the mixin once (ie the best-available version). See: https://github.com/totten/civix/issues/175 Before ------ The civix templates generate a few files, such as `mymod.php` and `mymod.civix.php`. A typical example looks like this: ```php // mymod.php - Implement hook_civicrm_xmlMenu require_once 'mymod.civix.php'; function mymod_civicrm_xmlMenu(&$all, $the, $params) { _mymod_civix_civicrm_xmlMenu($all, $the, $params); } ``` and ```php // mymod.civix.php - Implement hook_civicrm_xmlMenu function _mymod_civix_civicrm_xmlMenu(&$all, $the, $params) { foreach (_mosaico_civix_glob(__DIR__ . '/xml/Menu/*.xml') as $file) { $files[] = $file; } } ``` These two files are managed differently: `mymod.php` is owned by the developer, and they may add/remove/manage the hooks in this file. `mymod.civix.php` is owned by `civix` and must be autogenerated. This structure allows `civix` (and any `civix`-based extension) to take advantage of new coding-convention immediately. However, it comes with a few pain-points: * If you want to write a patch for `_mymod_civix_civicrm_xmlMenu`, the dev-test-loop requires several steps. * If `civix` needs to add a new `hook_civicrm_foo`, then the author must manually create the stub function in `mymod.php`. `civix` has documentation (`UPGRADE.md`) which keeps a long list of stubs that must be manually added. * If `civix` has an update for `_mymod_civix_civicrm_xmlMenu`, then the author must regenerate `mymod.civix.php`. * If `mymod_civix_xmlMenu` needs a change, then the author must apply it manually. * If `civix`'s spin on `hook_civicrm_xmlMenu` becomes widespread, then the `xmlMenu` boilerplate is duplicated across many extensions. After ----- An extension may enable a mixin in `info.xml`, eg: ```xml civix-register-files@2.0 ``` Civi will look for a file `mixin/civicrm-register-files@2.0.0.mixin.php` (either in the extension or core). The file follows this pattern: ```php return function(\CRM_Extension_MixInfo $mixInfo, \CRM_Extension_BootCache $bootCache) { // echo "This is " . $mixInfo->longName . "!\n"; \Civi::dispatcher()->addListener("hook_civicrm_xmlMenu", function($e) use ($mixInfo) { ... }); } ``` The mixin file is a plain PHP file that can be debugged/copied/edited verbatim, and it can register for hooks on its own. The code is no longer a "template", and it doesn't need to be interwoven between `mymod.php` and `mymod.civix.php`. It is expected that a system may have multiple copies of a mixin. It will choose the newest compatible copy. Hypothetically, if there were a security update or internal API change, core might ship a newer version to supplant the old copy in any extensions. Technical Details ----------------- Mixins may define internal classes/interfaces/functions. However, each major-version must have a distinct prefix (e.g. `\V2\Mymixin\FooInterface`). Minor-versions may be provide incremental revisions over the same symbol (but it's imperative for newer increments to provide the backward-compatibility). MixinScanner - Make it easier to instantiate and pay with instances Ex: cv ev '$o=new CRM_Extension_MixinScanner(); var_export($o->createLoader());' MixinScanner - Enable scanning of `[civicrm.root]/mixin` --- CRM/Extension/BootCache.php | 85 ++++++++++++++ CRM/Extension/ClassLoader.php | 39 ++++--- CRM/Extension/Info.php | 13 +++ CRM/Extension/MixInfo.php | 73 ++++++++++++ CRM/Extension/MixinLoader.php | 207 +++++++++++++++++++++++++++++++++ CRM/Extension/MixinScanner.php | 148 +++++++++++++++++++++++ 6 files changed, 551 insertions(+), 14 deletions(-) create mode 100644 CRM/Extension/BootCache.php create mode 100644 CRM/Extension/MixInfo.php create mode 100644 CRM/Extension/MixinLoader.php create mode 100644 CRM/Extension/MixinScanner.php diff --git a/CRM/Extension/BootCache.php b/CRM/Extension/BootCache.php new file mode 100644 index 0000000000..574a32dd74 --- /dev/null +++ b/CRM/Extension/BootCache.php @@ -0,0 +1,85 @@ +define('initTime', function() { return time(); }); + * + * @param string $key + * @param mixed $callback + * @return mixed + * The value of $callback (either cached or fresh) + */ + public function define($key, $callback) { + if (!isset($this->data[$key])) { + $this->set($key, $callback($this)); + } + return $this->data[$key]; + } + + /** + * Determine if $key has been set. + * + * @param string $key + * @return bool + */ + public function has($key) { + return isset($this->data[$key]); + } + + /** + * Get the value of $key. + * + * @param string $key + * @param mixed $default + * @return mixed + */ + public function get($key, $default = NULL) { + return $this->data[$key] ?? $default; + } + + /** + * Set a value in the cache. + * + * This operation is only valid on the first page-load when a cache is built. + * + * @param string $key + * @param mixed $value + * @return static + * @throws \Exception + */ + public function set($key, $value) { + if ($this->locked) { + throw new \Exception("Cannot modify a locked boot-cache."); + } + $this->data[$key] = $value; + return $this; + } + + public function lock() { + $this->locked = TRUE; + } + +} diff --git a/CRM/Extension/ClassLoader.php b/CRM/Extension/ClassLoader.php index a46580651d..403c682f08 100644 --- a/CRM/Extension/ClassLoader.php +++ b/CRM/Extension/ClassLoader.php @@ -63,23 +63,33 @@ class CRM_Extension_ClassLoader { */ public function register() { // In pre-installation environments, don't bother with caching. - if (!defined('CIVICRM_DSN') || defined('CIVICRM_TEST') || \CRM_Utils_System::isInUpgradeMode()) { - $this->loader = $this->buildClassLoader(); - return $this->loader->register(); - } + $cacheFile = (defined('CIVICRM_DSN') && !defined('CIVICRM_TEST') && !\CRM_Utils_System::isInUpgradeMode()) + ? $this->getCacheFile() : NULL; - $file = $this->getCacheFile(); - if (file_exists($file)) { - $this->loader = require $file; + if (file_exists($cacheFile)) { + [$classLoader, $mixinLoader, $bootCache] = require $cacheFile; + $cacheUpdate = NULL; } else { - $this->loader = $this->buildClassLoader(); - $ser = serialize($this->loader); - file_put_contents($file, - sprintf("buildClassLoader(); + $mixinLoader = (new CRM_Extension_MixinScanner($this->mapper, $this->manager, $cacheFile !== NULL))->createLoader(); + $bootCache = new CRM_Extension_BootCache(); + // We don't own Composer\Autoload\ClassLoader, so we clone to prevent register() from potentially leaking data. + // We do own MixinLoader, and we want its state - like $bootCache - to be written. + $cacheUpdate = $cacheFile ? [clone $classLoader, clone $mixinLoader, $bootCache] : NULL; } - return $this->loader->register(); + + $classLoader->register(); + $mixinLoader->run($bootCache); + + if ($cacheUpdate !== NULL) { + // Save cache after $mixinLoader has a chance to fill $bootCache. + $export = var_export(serialize($cacheUpdate), 1); + file_put_contents($cacheFile, sprintf("loader = $classLoader; + return $classLoader; } /** @@ -162,7 +172,8 @@ class CRM_Extension_ClassLoader { * @return string */ protected function getCacheFile() { - $envId = \CRM_Core_Config_Runtime::getId(); + $formatRev = '_2'; + $envId = \CRM_Core_Config_Runtime::getId() . $formatRev; $file = \Civi::paths()->getPath("[civicrm.compile]/CachedExtLoader.{$envId}.php"); return $file; } diff --git a/CRM/Extension/Info.php b/CRM/Extension/Info.php index 8b8ee51143..2f14314c07 100644 --- a/CRM/Extension/Info.php +++ b/CRM/Extension/Info.php @@ -44,6 +44,13 @@ class CRM_Extension_Info { */ public $requires = []; + /** + * @var array + * List of expected mixins. + * Ex: ['civix@2.0.0'] + */ + public $mixins = []; + /** * @var array * List of strings (tag-names). @@ -197,6 +204,12 @@ class CRM_Extension_Info { $this->tags[] = (string) $tag; } } + elseif ($attr === 'mixins') { + $this->mixins = []; + foreach ($val->mixin as $mixin) { + $this->mixins[] = (string) $mixin; + } + } elseif ($attr === 'requires') { $this->requires = $this->filterRequirements($val); } diff --git a/CRM/Extension/MixInfo.php b/CRM/Extension/MixInfo.php new file mode 100644 index 0000000000..4e97d8f8e9 --- /dev/null +++ b/CRM/Extension/MixInfo.php @@ -0,0 +1,73 @@ +civix@1.0`). + * - They may be copied/reproduced in multiple extensions. + * - They are de-duped - such that a major-version (eg `civix@1` or `civix@2`) is only loaded once. + * + * The "MixInfo" record tracks the mixins needed by an extension. You may consider this an + * optimized subset of the 'info.xml'. (The mix-info is loaded on every page-view, so this + * record is serialized and stored in the MixinLoader cache.) + */ +class CRM_Extension_MixInfo { + + /** + * @var string + * + * Ex: 'org.civicrm.flexmailer' + */ + public $longName; + + /** + * @var string + * + * Ex: 'flexmailer' + */ + public $shortName; + + /** + * @var string|null + * + * Ex: '/var/www/modules/civicrm/ext/flexmailer'. + */ + public $path; + + /** + * @var array + * Ex: ['civix@2.0', 'menu@1.0'] + */ + public $mixins; + + /** + * Get a path relative to the target extension. + * + * @param string $relPath + * @return string + */ + public function getPath($relPath = NULL) { + return $relPath === NULL ? $this->path : $this->path . DIRECTORY_SEPARATOR . ltrim($relPath, '/'); + } + + public function isActive() { + return \CRM_Extension_System::singleton()->getMapper()->isActiveModule($this->shortName); + } + +} diff --git a/CRM/Extension/MixinLoader.php b/CRM/Extension/MixinLoader.php new file mode 100644 index 0000000000..2b8d5277be --- /dev/null +++ b/CRM/Extension/MixinLoader.php @@ -0,0 +1,207 @@ + 'path/to/civix@1.0.0.mixin.php'] + */ + protected $liveFuncFiles = NULL; + + /** + * @var array + * Ex: ['civix' => ['1.0.0' => 'path/to/civix@1.0.0.mixin.php']] + */ + protected $allFuncFiles = []; + + /** + * @param CRM_Extension_MixInfo $mix + * @return static + * @throws \CRM_Extension_Exception_ParseException + */ + public function addMixInfo(CRM_Extension_MixInfo $mix) { + $this->mixInfos[$mix->longName] = $mix; + return $this; + } + + /** + * @param array|string $files + * Ex: 'path/to/some/file@1.0.0.mixin.php' + * @param bool $deepRead + * If TRUE, then the file will be read to find metadata. + * @return $this + */ + public function addFunctionFiles($files, $deepRead = FALSE) { + $files = (array) $files; + foreach ($files as $file) { + if (preg_match(';^([^@]+)@([^@]+)\.mixin\.php$;', basename($file), $m)) { + $this->allFuncFiles[$m[1]][$m[2]] = $file; + continue; + } + + if ($deepRead) { + $header = $this->loadFunctionFileHeader($file); + if (isset($header['mixinName'], $header['mixinVersion'])) { + $this->allFuncFiles[$header['mixinName']][$header['mixinVersion']] = $file; + continue; + } + else { + error_log(sprintf('MixinLoader: Invalid mixin header for "%s". @mixinName and @mixinVersion required.', $file)); + continue; + } + } + + error_log(sprintf('MixinLoader: File \"%s\" cannot be parsed.', $file)); + } + return $this; + } + + private function loadFunctionFileHeader($file) { + $php = file_get_contents($file, TRUE); + foreach (token_get_all($php) as $token) { + if (is_array($token) && in_array($token[0], [T_DOC_COMMENT, T_COMMENT, T_FUNC_C, T_METHOD_C, T_TRAIT_C, T_CLASS_C])) { + return \Civi\Api4\Utils\ReflectionUtils::parseDocBlock($token[1]); + } + } + return []; + } + + /** + * Optimize the metadata, removing information that is not needed at runtime. + * + * Steps: + * + * - Remove any unnecessary $mixInfos (ie they have no mixins). + * - Given the available versions and expectations, pick the best $liveFuncFiles. + * - Drop $allFuncFiles. + */ + public function compile() { + $this->liveFuncFiles = []; + $allFuncs = $this->allFuncFiles ?? []; + + $sortByVer = function ($a, $b) { + return version_compare($a, $b /* ignore third arg */); + }; + foreach (array_keys($allFuncs) as $name) { + uksort($allFuncs[$name], $sortByVer); + } + + $this->mixInfos = array_filter($this->mixInfos, function(CRM_Extension_MixInfo $mixInfo) { + return !empty($mixInfo->mixins); + }); + + foreach ($this->mixInfos as $ext) { + /** @var \CRM_Extension_MixInfo $ext */ + foreach ($ext->mixins as $verExpr) { + list ($name, $expectVer) = explode('@', $verExpr); + $matchFile = NULL; + // NOTE: allFuncs[$name] is sorted by increasing version number. Choose highest satisfactory match. + foreach ($allFuncs[$name] ?? [] as $availVer => $availFile) { + if (static::satisfies($expectVer, $availVer)) { + $matchFile = $availFile; + } + } + if ($matchFile) { + $this->liveFuncFiles[$verExpr] = $matchFile; + } + else { + error_log(sprintf('MixinLoader: Failed to locate match for "%s"', $verExpr)); + } + } + } + + $this->allFuncFiles = NULL; + + return $this; + } + + /** + * Load all extensions and call their respective function-files. + * + * @return static + * @throws \CRM_Core_Exception + */ + public function run(CRM_Extension_BootCache $bootCache) { + if ($this->liveFuncFiles === NULL) { + throw new CRM_Core_Exception("Premature initialization. MixinLoader has not identified live functions."); + } + + // == WIP == + // + //Do mixins run strictly once (during boot)? Or could they run twice? Or incrementally? Some edge-cases: + // - Mixins should make changes via dispatcher() and container(). If there's a Civi::reset(), then these things go away. We'll need to + // re-register. (Example scenario: unit-testing) + // - Mixins register for every active module. If a new module is enabled, then we haven't had a chance to run on the new extension. + // - Mixins register for every active module. If an old module is disabled, then there may be old listeners/services lingering. + if (!isset(\Civi::$statics[__CLASS__]['done'])) { + \Civi::$statics[__CLASS__]['done'] = []; + } + $done = &\Civi::$statics[__CLASS__]['done']; + + // Read each live func-file once, even if there's some kind of Civi::reset(). This avoids hard-crash where the func-file registers a PHP class/function/interface. + // Granted, PHP symbols require care to avoid conflicts between `mymixin@1.0` and `mymixin@2.0` -- but you can deal with that. For minor-versions, you're + // safe because we deduplicate. + static $funcsByFile = []; + foreach ($this->liveFuncFiles as $verExpr => $file) { + if (!isset($funcsByFile[$file])) { + $func = include_once $file; + if (is_callable($func)) { + $funcsByFile[$file] = $func; + } + else { + error_log(sprintf('MixinLoader: Received invalid callback from \"%s\"', $file)); + } + } + } + + foreach ($this->mixInfos as $ext) { + /** @var \CRM_Extension_MixInfo $ext */ + foreach ($ext->mixins as $verExpr) { + $doneId = $ext->longName . '::' . $verExpr; + if (isset($done[$doneId])) { + continue; + } + if (isset($funcsByFile[$this->liveFuncFiles[$verExpr]])) { + call_user_func($funcsByFile[$this->liveFuncFiles[$verExpr]], $ext, $bootCache); + $done[$doneId] = 1; + } + else { + error_log(sprintf('MixinLoader: Failed to load "%s" for extension "%s"', $verExpr, $ext->longName)); + } + } + } + + return $this; + } + + /** + * @param string $expectVer + * @param string $actualVer + * @return bool + */ + private static function satisfies($expectVer, $actualVer) { + [$expectMajor] = explode('.', $expectVer); + [$actualMajor] = explode('.', $actualVer); + return ($expectMajor == $actualMajor) && version_compare($actualVer, $expectVer, '>='); + } + +} diff --git a/CRM/Extension/MixinScanner.php b/CRM/Extension/MixinScanner.php new file mode 100644 index 0000000000..433f647d0b --- /dev/null +++ b/CRM/Extension/MixinScanner.php @@ -0,0 +1,148 @@ +mapper = $mapper ?: CRM_Extension_System::singleton()->getMapper(); + $this->manager = $manager ?: CRM_Extension_System::singleton()->getManager(); + if ($relativize) { + $this->relativeBases = [Civi::paths()->getVariable('civicrm.root', 'path')]; + // Previous drafts used `relativeBases=explode(include_path)`. However, this produces unstable results + // when flip through the phases of the lifecycle - because the include_path changes throughout the lifecycle. + usort($this->relativeBases, function($a, $b) { + return strlen($b) - strlen($a); + }); + } + else { + $this->relativeBases = NULL; + } + } + + /** + * @return \CRM_Extension_MixinLoader + */ + public function createLoader() { + $l = new CRM_Extension_MixinLoader(); + + foreach ($this->getInstalledKeys() as $key) { + try { + $path = $this->mapper->keyToBasePath($key); + $l->addMixInfo($this->createMixInfo($path . DIRECTORY_SEPARATOR . CRM_Extension_Info::FILENAME)); + $l->addFunctionFiles($this->findFunctionFiles("$path/mixin/*@*.mixin.php")); + $l->addFunctionFiles($this->findFunctionFiles("$path/mixin/*@*/mixin.php"), TRUE); + } + catch (CRM_Extension_Exception_ParseException $e) { + error_log(sprintf('MixinScanner: Failed to read extension (%s)', $key)); + } + } + + $l->addFunctionFiles($this->findFunctionFiles(Civi::paths()->getPath('[civicrm.root]/mixin/*@*.mixin.php'))); + $l->addFunctionFiles($this->findFunctionFiles(Civi::paths()->getPath('[civicrm.root]/mixin/*@*/mixin.php')), TRUE); + + return $l->compile(); + } + + /** + * @return array + */ + private function getInstalledKeys() { + $keys = []; + + $statuses = $this->manager->getStatuses(); + ksort($statuses); + foreach ($statuses as $key => $status) { + if ($status === CRM_Extension_Manager::STATUS_INSTALLED) { + $keys[] = $key; + } + } + + return $keys; + } + + /** + * @param string $infoFile + * Path to the 'info.xml' file + * @return \CRM_Extension_MixInfo + * @throws \CRM_Extension_Exception_ParseException + */ + private function createMixInfo(string $infoFile) { + $info = CRM_Extension_Info::loadFromFile($infoFile); + $instance = new CRM_Extension_MixInfo(); + $instance->longName = $info->key; + $instance->shortName = $info->file; + $instance->path = rtrim(dirname($infoFile), '/' . DIRECTORY_SEPARATOR); + $instance->mixins = $info->mixins; + return $instance; + } + + /** + * @param string $globPat + * @return array + * Ex: ['mix/xml-menu-autoload@1.0.mixin.php'] + */ + private function findFunctionFiles($globPat) { + $useRel = $this->relativeBases !== NULL; + $result = []; + $funcFiles = (array) glob($globPat); + sort($funcFiles); + foreach ($funcFiles as $shimFile) { + $shimFileRel = $useRel ? $this->relativize($shimFile) : $shimFile; + $result[] = $shimFileRel; + } + return $result; + } + + /** + * Convert the absolute $file to an expression that is supported by 'include'. + * + * @param string $file + * @return string + */ + private function relativize($file) { + foreach ($this->relativeBases as $relativeBase) { + if (CRM_Utils_File::isChildPath($relativeBase, $file)) { + return ltrim(CRM_Utils_File::relativize($file, $relativeBase), '/' . DIRECTORY_SEPARATOR); + } + } + return $file; + } + +} -- 2.25.1