From b233b6caf4953665a9dd865006e3132a9dbd93d3 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Tue, 30 Nov 2021 12:14:10 -0800 Subject: [PATCH] tools/mixin - Import. Also, update to run within civicrm-core, and add JUnit output. --- tools/mixin/bin/mixer | 279 ++++++++++++++++++++++++++++++++++++ tools/mixin/bin/test-all | 58 ++++++++ tools/mixin/src/Mixlib.php | 284 +++++++++++++++++++++++++++++++++++++ 3 files changed, 621 insertions(+) create mode 100755 tools/mixin/bin/mixer create mode 100755 tools/mixin/bin/test-all create mode 100644 tools/mixin/src/Mixlib.php diff --git a/tools/mixin/bin/mixer b/tools/mixin/bin/mixer new file mode 100755 index 0000000000..e7207adc98 --- /dev/null +++ b/tools/mixin/bin/mixer @@ -0,0 +1,279 @@ +#!/usr/bin/env php + TRUE] + * @param string $targetDir + * The output directory. + * @param mixed ...$mixinNames + * List of mixins to include in the generated extension. + * @return string + */ +function task_create(array $options, string $targetDir, ...$mixinNames) { + if (file_exists($targetDir)) { + if (!empty($options['force'])) { + fprintf(STDOUT, "Remove %s\n", $targetDir); + remove_dir($targetDir); + } + else { + throw new \RuntimeException("Cannot overwrite $targetDir"); + } + } + + $mixinNames = resolve_mixin_names($mixinNames); + fprintf(STDOUT, "Create %s for %s\n", $targetDir, implode(',', $mixinNames)); + + $srcDirs = []; + $srcDirs[] = mixer_shimmy_dir(); + foreach ($mixinNames as $mixinName) { + if (is_dir(mixer_mixlib_dir() . "/$mixinName/example")) { + $srcDirs[] = mixer_mixlib_dir() . "/$mixinName/example"; + } + } + deep_copy($srcDirs, $targetDir); + + $mixins = []; + foreach ($mixinNames as $mixinName) { + $mixins[$mixinName] = mixlib()->assertValid(mixlib()->get($mixinName)); + } + + if (empty($options['bare'])) { + mkdir("$targetDir/mixin"); + file_put_contents("$targetDir/mixin/polyfill.php", mixlib()->get('polyfill')['src']); + foreach ($mixins as $mixin) { + file_put_contents("$targetDir/mixin/{$mixin['mixinName']}@{$mixin['mixinVersion']}.mixin.php", $mixin['src']); + } + } + + rename(assert_file("$targetDir/info.xml.template"), "$targetDir/info.xml"); + update_xml("$targetDir/info.xml", function (SimpleXMLElement $info) use ($mixins) { + $mixinsXml = $info->addChild('mixins'); + foreach ($mixins as $mixinName => $mixin) { + $mixinsXml->addChild('mixin', $mixin['mixinName'] . '@' . $mixin['mixinVersion']); + } + if (!empty($options['bare'])) { + // If the example doesn't include mixins, then we must get them from elsewhere. + $requiresXml = $info->addChild('requires'); + $requiresXml->addChild('ext', 'mixinlib'); + } + }); + + return $targetDir; +} + +/** + * Generate and test an extension that uses the given mixins.. + * + * @param array $options + * Ex: ['force' => TRUE] + * Ex: ['isolate' => TRUE] + * @param string $targetDir + * The output directory. + * @param mixed ...$mixinNames + * List of mixins to include in the generated extension. + */ +function task_test(array $options, string $targetDir, ...$args) { + if (($split = array_search('--', $args)) !== FALSE) { + $mixinNames = array_slice($args, 0, $split); + $phpunitArgs = array_slice($args, $split + 1); + } + else { + $mixinNames = $args; + $phpunitArgs = explode(' ', '--group e2e --debug --stop-on-failure'); + } + + $mixinNames = resolve_mixin_names($mixinNames); + $errors = []; + if (!empty($options['isolate']) && count($mixinNames) > 1) { + foreach ($mixinNames as $mixinName) { + try { + task_test($options + ['force' => TRUE], $targetDir, $mixinName, '--', ...$phpunitArgs); + } + catch (\Throwable $t) { + fprintf(STDERR, "Error testing $mixinName\n%s\n\n", $t->getTraceAsString()); + $errors = [$mixinName]; + } + } + if ($errors) { + fprintf(STDERR, "Error processing mixins: %s\n", implode(' ', $errors)); + exit(1); + } + return; + } + + if (is_dir($targetDir) || !empty($options['force'])) { + $targetDir = task_create($options, $targetDir, ...$mixinNames); + } + if (empty(glob("$targetDir/tests/mixin/*.php"))) { + fprintf(STDERR, "Skip. No tests found for %s\n", implode(',', $mixinNames)); + return; + } + fprintf(STDOUT, "Test %s\n", implode(',', $mixinNames)); + with_dir($targetDir, function () use ($phpunitArgs) { + phpunit($phpunitArgs); + }); +} + +function task_list(array $options, ...$mixinNames) { + $mixinNames = resolve_mixin_names($mixinNames); + foreach ($mixinNames as $mixinName) { + $mixin = mixlib()->get($mixinName); + fprintf(STDOUT, "%-20s %-10s %s\n", $mixin['mixinName'], $mixin['mixinVersion'] ?? '', $mixin['description'] ?? ''); + } +} + +function task_help(array $options) { + $cmd = basename($GLOBALS['argv'][0]); + fprintf(STDERR, "%s - Test utility for extension mixins\n", $cmd); + fprintf(STDERR, "\n"); + fprintf(STDERR, "Usage:\n"); + fprintf(STDERR, " %s create [-f] [--bare] [...]\n", $cmd); + fprintf(STDERR, " %s test [-f] [--bare] [--isolate] [...] -- [...]\n", $cmd); + fprintf(STDERR, " %s list [...]\n", $cmd); +} + +function mixer_mixlib_dir(): string { + return dirname(__DIR__, 3) . '/mixin'; +} + +function mixer_shimmy_dir(): string { + return dirname(__DIR__, 3) . '/tests/extensions/shimmy'; +} + +function assert_dir(string $dir): string { + if (!file_exists($dir) || !is_dir($dir)) { + throw new \RuntimeException("Directory does not exist ($dir)"); + } + return $dir; +} + +function assert_file(string $file): string { + if (!file_exists($file)) { + throw new \RuntimeException("File does not exist ($file)"); + } + return $file; +} + +function remove_dir(string $dir): void { + if (file_exists($dir)) { + passthru_ok("rm -rf " . escapeshellarg($dir)); + } +} + +function deep_copy(array $srcDirs, string $targetDir): void { + $srcDirs = (array) $srcDirs; + foreach ($srcDirs as $srcDir) { + assert_dir($srcDir); + } + + if (!file_exists($targetDir)) { + mkdir($targetDir); + } + + foreach ($srcDirs as $srcDir) { + passthru_ok(sprintf('rsync -a %s/./ %s/./', escapeshellarg($srcDir), escapeshellarg($targetDir))); + } +} + +function phpunit(array $args = []) { + $argString = implode(' ' , array_map('escapeshellarg', $args)); + passthru_ok('phpunit8 ' . $argString); +} + +function passthru_ok($cmd) { + passthru($cmd, $return); + if ($return !== 0) { + throw new \RuntimeException("Command failed ($cmd)"); + } +} + +function resolve_mixin_names(array $mixinNames): array { + if (empty($mixinNames)) { + return mixlib()->getList(); + } + else { + return array_map( + function (string $mixinName) { + return trim($mixinName, '/' . DIRECTORY_SEPARATOR); + }, $mixinNames); + } +} + +function with_dir(string $dir, callable $callback) { + assert_dir($dir); + $orig = getcwd(); + try { + chdir($dir); + $callback(); + } finally { + chdir($orig); + } +} + +function update_xml(string $file, callable $filter): void { + $dom = new DomDocument(); + $dom->load($file); + $dom->xinclude(); + $simpleXml = simplexml_import_dom($dom); + $filter($simpleXml); + // force pretty printing with encode/decode cycle + $outXML = $simpleXml->saveXML(); + $xml = new DOMDocument(); + $xml->encoding = 'iso-8859-1'; + $xml->preserveWhiteSpace = FALSE; + $xml->formatOutput = TRUE; + $xml->loadXML($outXML); + file_put_contents($file, $xml->saveXML()); +} + +function main($args) { + $cmd = array_shift($args); + + $isParsingOptions = TRUE; + $newOptions = $newArgs = []; + $task = NULL; + foreach ($args as $arg) { + if (!$isParsingOptions) { + $newArgs[] = $arg; + } + elseif ($arg === '--') { + $isParsingOptions = FALSE; + $newArgs[] = $arg; + } + elseif ($arg === '-f') { + $newOptions['force'] = TRUE; + } + elseif (preg_match(';^--(bare|isolate|force)$;', $arg, $m)) { + $newOptions[$m[1]] = TRUE; + } + elseif ($task === NULL) { + $task = 'task_' . preg_replace(';[^\w];', '_', $arg); + } + else { + $newArgs[] = $arg; + } + } + + if (function_exists($task)) { + call_user_func($task, $newOptions, ...$newArgs); + } + else { + task_help([]); + exit(1); + } +} + +main($argv); diff --git a/tools/mixin/bin/test-all b/tools/mixin/bin/test-all new file mode 100755 index 0000000000..8a43b57984 --- /dev/null +++ b/tools/mixin/bin/test-all @@ -0,0 +1,58 @@ +#!/bin/bash +set -e +export XDEBUG_PORT= XDEBUG_MODE=off + +############################################################################### +## Bootstrap + +## Determine the absolute path of the directory with the file +## usage: absdirname +function absdirname() { + pushd $(dirname $0) >> /dev/null + pwd + popd >> /dev/null +} + +SCRIPT_DIR=$(absdirname "$0") +EXT_DIR=$(cv path -c extensionsDir)/example-mixin +JUNIT_DIR="$1" + +## TODO: Once the managed-entity regression is examined/fixed, remove the MY_MIXINS list. Then it will test all mixins. +# MY_MIXINS='ang-php@1 case-xml@1 menu-xml@1 setting-php@1 theme-php@1' +MY_MIXINS='' + +############################################################################### + +## usage: mixer_test [--bare] [--isolate] ... +function mixer_test() { + local XML_FILE="$1" + shift + + [ -f "$JUNIT_DIR/$XML_FILE" ] && rm -f "$JUNIT_DIR/$XML_FILE" + + civibuild restore + cv flush + + ## usage: mixer test [-f] [--bare] [--isolate] [...] -- [...] + "$SCRIPT_DIR/mixer" test -f "$EXT_DIR" "$@" -- --group e2e --log-junit "$JUNIT_DIR/$XML_FILE" +} + +############################################################################### + +if [ -z "$EXT_DIR" -o ! -e "$EXT_DIR" ]; then + echo "Invalid extension dir: $EXT_DIR" + exit 1 +fi + +if [ -z "$JUNIT_DIR" ]; then + echo "Missing argument: " + exit 1 +elif [ ! -d "$JUNIT_DIR" ]; then + mkdir -p "$JUNIT_DIR" +fi + + +set -ex +mixer_test "mixin-isolate-bare.xml" $MY_MIXINS --isolate --bare +mixer_test "mixin-combine-bare.xml" $MY_MIXINS --bare +mixer_test "mixin-combine-copy.xml" $MY_MIXINS diff --git a/tools/mixin/src/Mixlib.php b/tools/mixin/src/Mixlib.php new file mode 100644 index 0000000000..d2ba33d869 --- /dev/null +++ b/tools/mixin/src/Mixlib.php @@ -0,0 +1,284 @@ +getPath('[civicrm.root]/mixin') + * @param string|NULL $mixlibUrl + * Ex: "https://raw.githubusercontent.com/civicrm/civicrm-core/5.45/mixin" + * Ex: "https://raw.githubusercontent.com/totten/civicrm-core/master-mix-dec/mixin" + */ + public function __construct(?string $mixlibDir = NULL, ?string $mixlibUrl = NULL) { + $this->mixlibDir = $mixlibDir ?: dirname(__DIR__, 3) . '/mixin'; + $this->mixlibUrl = $mixlibUrl; + } + + public function getList(): array { + if ($this->mixlibDir === NULL || !file_exists($this->mixlibDir)) { + throw new \RuntimeException("Cannot get list of available mixins"); + } + + if (isset($this->cache['getList'])) { + return $this->cache['getList']; + } + + $dirs = (array) glob($this->mixlibDir . '/*@*'); + $mixinNames = []; + foreach ($dirs as $dir) { + if (is_dir($dir)) { + $mixinNames[] = basename($dir); + } + } + sort($mixinNames); + $this->cache['getList'] = $mixinNames; + return $mixinNames; + } + + /** + * @param string $mixin + * + * @return array + * Item with keys: + * - mixinName: string, eg 'mgd-php' + * - mixinVersion: string, eg '1.0.2' + * - mixinConstraint: string, eg 'mgd-php@1.0.2' + * - mixinFile: string, eg 'mgd-php@1.0.2.mixin.php' + * - src: string, unevaluated PHP source + */ + public function get(string $mixin) { + if (isset($this->cache["parsed:$mixin"])) { + return $this->cache["parsed:$mixin"]; + } + + $phpCode = $this->getSourceCode($mixin); + $mixinSpec = $this->parseString($phpCode); + $mixinSpec['mixinName'] = $mixinSpec['mixinName'] ?? preg_replace(';@.*$;', '', $mixin); + + $parts = explode('@', $mixin); + $effectiveVersion = !empty($mixinSpec['mixinVersion']) ? $mixinSpec['mixinVersion'] : ($parts[1] ?? ''); + if ($effectiveVersion) { + $mixinSpec = array_merge([ + 'mixinConstraint' => $mixinSpec['mixinName'] . '@' . $effectiveVersion, + 'mixinFile' => $mixinSpec['mixinName'] . '@' . $effectiveVersion . '.mixin.php', + ], $mixinSpec); + } + else { + $mixinSpec = array_merge([ + 'mixinFile' => $mixinSpec['mixinName'] . '.mixin.php', + ], $mixinSpec); + } + $mixinSpec['src'] = $phpCode; + $this->cache["parsed:$mixin"] = $mixinSpec; + + return $this->cache["parsed:$mixin"]; + } + + /** + * Consolidate and retrieve the listed mixins. + * + * @param array $mixinConstraints + * Ex: ['foo@1.0', 'bar@1.2', 'bar@1.3'] + * @return array + * Ex: ['foo@1.0' => array, 'bar@1.3' => array] + */ + public function consolidate(array $mixinConstraints): array { + // Find and remove duplicate constraints. Pick tightest constraint. + // array(string $mixinName => string $mixinVersion) + $preferredVersions = []; + foreach ($mixinConstraints as $mixinName) { + [$name, $version] = explode('@', $mixinName); + if (!isset($preferredVersions[$name])) { + $preferredVersions[$name] = $version; + } + elseif (version_compare($version, $preferredVersions[$name], '>=')) { + $preferredVersions[$name] = $version; + } + } + + // Resolve current versions matching constraint. + $result = []; + foreach ($preferredVersions as $mixinName => $mixinVersion) { + $result[] = $mixinName . '@' . $mixinVersion; + } + sort($result); + return $result; + } + + /** + * Consolidate and retrieve the listed mixins. + * + * @param array $mixinConstraints + * Ex: ['foo@1.0', 'bar@1.2', 'bar@1.3'] + * @return array + * Ex: ['foo@1.0' => array, 'bar@1.3' => array] + */ + public function resolve(array $mixinConstraints): array { + $mixinConstraints = $this->consolidate($mixinConstraints); + + $result = []; + foreach ($mixinConstraints as $mixinConstraint) { + [$expectName, $expectVersion] = explode('@', $mixinConstraint); + $mixin = $this->get($mixinConstraint); + $this->assertValid($mixin); + if (!version_compare($mixin['mixinVersion'], $expectVersion, '>=') || $mixin['mixinName'] !== $expectName) { + throw new \RuntimeException(sprintf("Received incompatible version (expected=\"%s@%s\", actual=\"%s@%s\")", $expectName, $expectVersion, $mixin['mixinName'], $mixin['mixinVersion'])); + } + $result[$mixin['mixinConstraint']] = $mixin; + } + return $result; + } + + /** + * @param string $mixin + * Ex: 'foo@1.2.3', 'foo-bar@4.5.6', 'polyfill', + * @return string + */ + protected function getSourceCode(string $mixin): string { + if ($mixin === 'polyfill') { + $file = 'polyfill.php'; + } + elseif (preg_match(';^([-\w]+)@(\d+)([\.\d]+)?;', $mixin, $m)) { + // Get the last revision within the major series. + $file = sprintf('%s@%s/mixin.php', $m[1], $m[2]); + } + else { + throw new \RuntimeException("Failed to parse mixin name ($mixin)"); + } + + if ($this->mixlibDir && file_exists($this->mixlibDir . '/' . $file)) { + return file_get_contents($this->mixlibDir . '/' . $file); + } + + if ($this->mixlibUrl) { + $url = $this->mixlibUrl . '/' . $file; + $download = file_get_contents($url); + if (!empty($download)) { + $this->cache["src:$mixin"] = $download; + return $download; + } + } + + throw new \RuntimeException("Failed to locate $file (mixlibDir={$this->mixlibDir}, mixlibUrl={$this->mixlibUrl})"); + } + + public function assertValid(array $mixin): array { + if (empty($mixin['mixinVersion'])) { + throw new \RuntimeException("Invalid {$mixin["file"]}. There is no @mixinVersion annotation."); + } + if (empty($mixin['mixinVersion'])) { + throw new \RuntimeException("Invalid {$mixin["file"]}. There is no @mixinName annotation."); + } + return $mixin; + } + + /** + * @param string $phpCode + * @return array + */ + protected function parseString(string $phpCode): array { + $commmentTokens = [T_DOC_COMMENT, T_COMMENT, T_FUNC_C, T_METHOD_C, T_TRAIT_C, T_CLASS_C]; + $mixinSpec = []; + foreach (token_get_all($phpCode) as $token) { + if (is_array($token) && in_array($token[0], $commmentTokens)) { + $mixinSpec = $this->parseComment($token[1]); + break; + } + } + return $mixinSpec; + } + + protected function parseComment(string $comment): array { + $info = []; + $param = NULL; + foreach (preg_split("/((\r?\n)|(\r\n?))/", $comment) as $num => $line) { + if (!$num || strpos($line, '*/') !== FALSE) { + continue; + } + $line = ltrim(trim($line), '*'); + if (strlen($line) && $line[0] === ' ') { + $line = substr($line, 1); + } + if (strpos(ltrim($line), '@') === 0) { + $words = explode(' ', ltrim($line, ' @')); + $key = array_shift($words); + $param = NULL; + if ($key == 'var') { + $info['type'] = explode('|', $words[0]); + } + elseif ($key == 'return') { + $info['return'] = explode('|', $words[0]); + } + elseif ($key == 'options' || $key == 'ui_join_filters') { + $val = str_replace(', ', ',', implode(' ', $words)); + $info[$key] = explode(',', $val); + } + elseif ($key == 'throws' || $key == 'see') { + $info[$key][] = implode(' ', $words); + } + elseif ($key == 'param' && $words) { + $type = $words[0][0] !== '$' ? explode('|', array_shift($words)) : NULL; + $param = rtrim(array_shift($words), '-:()/'); + $info['params'][$param] = [ + 'type' => $type, + 'description' => $words ? ltrim(implode(' ', $words), '-: ') : '', + 'comment' => '', + ]; + } + else { + // Unrecognized annotation, but we'll duly add it to the info array + $val = implode(' ', $words); + $info[$key] = strlen($val) ? $val : TRUE; + } + } + elseif ($param) { + $info['params'][$param]['comment'] .= $line . "\n"; + } + elseif ($num == 1) { + $info['description'] = ucfirst($line); + } + elseif (!$line) { + if (isset($info['comment'])) { + $info['comment'] .= "\n"; + } + else { + $info['comment'] = NULL; + } + } + // For multi-line description. + elseif (count($info) === 1 && isset($info['description']) && substr($info['description'], -1) !== '.') { + $info['description'] .= ' ' . $line; + } + else { + $info['comment'] = isset($info['comment']) ? "{$info['comment']}\n$line" : $line; + } + } + if (isset($info['comment'])) { + $info['comment'] = rtrim($info['comment']); + } + return $info; + } + +} -- 2.25.1