INFRA-125, CRM-15011 - Import phpunit-{antagonist,compare,indiv,ls} and jmat

diff --git a/Civi/CiUtil/Arrays.php b/Civi/CiUtil/Arrays.php
new file mode 100644 (file)
index 0000000..359ad4c
--- /dev/null
+++ b/Civi/CiUtil/Arrays.php
@@ -0,0 +1,12 @@
+<?php
+namespace Civi\CiUtil;
+
+class Arrays {
+  static function collect($arr, $col) {
+    $r = array();
+    foreach ($arr as $k => $item) {
+      $r[$k] = $item[$col];
+    }
+    return $r;
+  }
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/Command/AntagonistCommand.php b/Civi/CiUtil/Command/AntagonistCommand.php
new file mode 100644 (file)
index 0000000..9a71959
--- /dev/null
+++ b/Civi/CiUtil/Command/AntagonistCommand.php
@@ -0,0 +1,67 @@
+<?php
+namespace Civi\CiUtil\Command;
+
+class AntagonistCommand {
+  static function main($argv) {
+    if (count($argv) != 3) {
+      print "usage: {$argv[0]} <TargetTest::testFunc> </path/to/suite>\n";
+      exit(1);
+    }
+    list ($program, $target, $suite) = $argv;
+
+    $candidateTests = \Civi\CiUtil\PHPUnitScanner::findTestsByPath(array($suite));
+//    $candidateTests = array(
+//      array('class' => 'CRM_Core_RegionTest', 'method' => 'testBlank'),
+//      array('class' => 'CRM_Core_RegionTest', 'method' => 'testDefault'),
+//      array('class' => 'CRM_Core_RegionTest', 'method' => 'testOverride'),
+//      array('class' => 'CRM_Core_RegionTest', 'method' => 'testAllTypes'),
+//    );
+    $antagonist = self::findAntagonist($target, $candidateTests);
+    if ($antagonist) {
+      print_r(array('found an antagonist' => $antagonist));
+    }
+    else {
+      print_r(array('found no antagonists'));
+    }
+  }
+
+  /**
+   * @param string $target e.g. "MyTest::testFoo"
+   * @param array $candidateTests list of strings (e.g. "MyTest::testFoo")
+   * @return array|null array contains keys:
+   *  - antagonist: array
+   *    - file: string
+   *    - class: string
+   *    - method: string
+   *  - expectedResults: array
+   *  - actualResults: array
+   */
+  static function findAntagonist($target, $candidateTests) {
+    //$phpUnit = new \Civi\CiUtil\EnvTestRunner('./scripts/phpunit', 'EnvTests');
+    $phpUnit = new \Civi\CiUtil\EnvTestRunner('phpunit', 'tests/phpunit/EnvTests.php');
+    $expectedResults = $phpUnit->run(array($target));
+    print_r(array('$expectedResults' => $expectedResults));
+
+    foreach ($candidateTests as $candidateTest) {
+      $candidateTestName = $candidateTest['class'] . '::' . $candidateTest['method'];
+      if ($candidateTestName == $target) {
+        continue;
+      }
+      $actualResults = $phpUnit->run(array(
+        $candidateTestName,
+        $target,
+      ));
+      print_r(array('$actualResults' => $actualResults));
+      foreach ($expectedResults as $testName => $expectedResult) {
+        if ($actualResults[$testName] != $expectedResult) {
+          return array(
+            'antagonist' => $candidateTest,
+            'expectedResults' => $expectedResults,
+            'actualResults' => $actualResults,
+          );
+        }
+      }
+    }
+    return NULL;
+  }
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/Command/CompareCommand.php b/Civi/CiUtil/Command/CompareCommand.php
new file mode 100644 (file)
index 0000000..494e909
--- /dev/null
+++ b/Civi/CiUtil/Command/CompareCommand.php
@@ -0,0 +1,42 @@
+<?php
+namespace Civi\CiUtil\Command;
+
+class CompareCommand {
+  static function main($argv) {
+    if (empty($argv[1])) {
+      echo "summary: Compares the output of different test runs\n";
+      echo "usage: phpunit-compare <json-file1> [<json-file2>...]\n";
+      exit(1);
+    }
+
+
+    $suites = array(); // array('file' => string, 'results' => array)
+    for ($i = 1; $i < count($argv); $i++) {
+      $suites[$i] = array(
+        'file' => $argv[$i],
+        'results' => \Civi\CiUtil\PHPUnitParser::parseJsonResults(file_get_contents($argv[$i]))
+      );
+    }
+
+    $tests = array(); // array(string $name)
+    foreach ($suites as $suiteName => $suite) {
+      $tests = array_unique(array_merge(
+        $tests,
+        array_keys($suite['results'])
+      ));
+    }
+    sort($tests);
+
+    $printer = new \Civi\CiUtil\ComparisonPrinter(\Civi\CiUtil\Arrays::collect($suites, 'file'));
+    foreach ($tests as $test) {
+      $values = array();
+      foreach ($suites as $suiteName => $suite) {
+        $values[] = isset($suite['results'][$test]) ? $suite['results'][$test] : 'MISSING';
+      }
+
+      if (count(array_unique($values)) > 1) {
+        $printer->printRow($test, $values);
+      }
+    }
+  }
+}

diff --git a/Civi/CiUtil/Command/LsCommand.php b/Civi/CiUtil/Command/LsCommand.php
new file mode 100644 (file)
index 0000000..b4af978
--- /dev/null
+++ b/Civi/CiUtil/Command/LsCommand.php
@@ -0,0 +1,12 @@
+<?php
+namespace Civi\CiUtil\Command;
+
+class LsCommand {
+  static function main($argv) {
+    $paths = $argv;
+    array_shift($paths);
+    foreach (\Civi\CiUtil\PHPUnitScanner::findTestsByPath($paths) as $test) {
+      printf("%s %s %s\n", $test['file'], $test['class'], $test['method']);
+    }
+  }
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/ComparisonPrinter.php b/Civi/CiUtil/ComparisonPrinter.php
new file mode 100644 (file)
index 0000000..60f5f2c
--- /dev/null
+++ b/Civi/CiUtil/ComparisonPrinter.php
@@ -0,0 +1,46 @@
+<?php
+namespace Civi\CiUtil;
+
+class ComparisonPrinter {
+  var $headers;
+  var $hasHeader = FALSE;
+
+  function __construct($headers) {
+    $this->headers = $headers;
+  }
+
+  function printHeader() {
+    if ($this->hasHeader) {
+      return;
+    }
+
+    ## LEGEND
+    print "LEGEND\n";
+    $i = 1;
+    foreach ($this->headers as $header) {
+      printf("% 2d: %s\n", $i, $header);
+      $i++;
+    }
+    print "\n";
+
+    ## HEADER
+    printf("%-90s ", 'TEST NAME');
+    $i = 1;
+    foreach ($this->headers as $header) {
+      printf("%-10d ", $i);
+      $i++;
+    }
+    print "\n";
+
+    $this->hasHeader = TRUE;
+  }
+
+  function printRow($test, $values) {
+    $this->printHeader();
+    printf("%-90s ", $test);
+    foreach ($values as $value) {
+      printf("%-10s ", $value);
+    }
+    print "\n";
+  }
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/EnvTestRunner.php b/Civi/CiUtil/EnvTestRunner.php
new file mode 100644 (file)
index 0000000..d16bd84
--- /dev/null
+++ b/Civi/CiUtil/EnvTestRunner.php
@@ -0,0 +1,31 @@
+<?php
+namespace Civi\CiUtil;
+
+/**
+ * Parse phpunit result files
+ */
+class EnvTestRunner {
+  protected $phpunit;
+  protected $envTestSuite;
+
+  function __construct($phpunit = "phpunit", $envTestSuite = 'EnvTests') {
+    $this->phpunit = $phpunit;
+    $this->envTestSuite = $envTestSuite;
+  }
+
+  /**
+   * @param array $tests
+   * @return array (string $testName => string $status)
+   */
+  public function run($tests) {
+    $envTests = implode(' ', $tests);
+    $jsonFile = tempnam(sys_get_temp_dir(), 'phpunit-json-');
+    unlink($jsonFile);
+    $command = "env PHPUNIT_TESTS=\"$envTests\" {$this->phpunit} --log-json $jsonFile {$this->envTestSuite}";
+    echo "Running [$command]\n";
+    system($command);
+    $results = PHPUnitParser::parseJsonResults(file_get_contents($jsonFile));
+    unlink($jsonFile);
+    return $results;
+  }
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/PHPUnitParser.php b/Civi/CiUtil/PHPUnitParser.php
new file mode 100644 (file)
index 0000000..bc28048
--- /dev/null
+++ b/Civi/CiUtil/PHPUnitParser.php
@@ -0,0 +1,34 @@
+<?php
+namespace Civi\CiUtil;
+
+/**
+ * Parse phpunit result files
+ */
+class PHPUnitParser {
+  /**
+   * @param string $content phpunit streaming JSON
+   * @return array(string "$class::$func" => $status)
+   */
+  protected static function parseJsonStream($content) {
+    $content = '['
+      . strtr($content, array("}{" => "},{"))
+      . ']';
+    return json_decode($content, TRUE);
+  }
+
+  /**
+   * @param string $content json stream
+   * @return array (string $testName => string $status)
+   */
+  public static function parseJsonResults($content) {
+    $records = self::parseJsonStream($content);
+    $results = array();
+    foreach ($records as $r) {
+      if ($r['event'] == 'test') {
+        $results[$r['test']] = $r['status'];
+      }
+    }
+    return $results;
+  }
+
+}
\ No newline at end of file

diff --git a/Civi/CiUtil/PHPUnitScanner.php b/Civi/CiUtil/PHPUnitScanner.php
new file mode 100644 (file)
index 0000000..b09de45
--- /dev/null
+++ b/Civi/CiUtil/PHPUnitScanner.php
@@ -0,0 +1,91 @@
+<?php
+namespace Civi\CiUtil;
+use Symfony\Component\Finder\Finder;
+
+/**
+ * Search for PHPUnit test cases
+ */
+class PHPUnitScanner {
+  /**
+   * @return array<string> class names
+   */
+  static function _findTestClasses($path) {
+//    print_r(array(
+//      'loading' => $path,
+//      get_included_files()
+//    ));
+
+    $origClasses = get_declared_classes();
+    require_once $path;
+    $newClasses = get_declared_classes();
+
+    return preg_grep('/Test$/', array_diff(
+      $newClasses,
+      $origClasses
+    ));
+  }
+
+  /**
+   * @return array (string $file => string $class)
+   */
+  static function findTestClasses($paths) {
+    $testClasses = array();
+    $finder = new Finder();
+
+    foreach ($paths as $path) {
+      if (is_dir($path)) {
+        foreach ($finder->files()->in($paths)->name('*Test.php') as $file) {
+          $testClass = self::_findTestClasses((string) $file);
+          if (count($testClass) == 1) {
+            $testClasses[(string) $file] = array_shift($testClass);
+          }
+          elseif (count($testClass) > 1) {
+            throw new \Exception("Too many classes in $file");
+          }
+          else {
+            throw new \Exception("Too few classes in $file");
+          }
+        }
+      }
+      elseif (is_file($path)) {
+        $testClass = self::_findTestClasses($path);
+        if (count($testClass) == 1) {
+          $testClasses[$path] = array_shift($testClass);
+        }
+        elseif (count($testClass) > 1) {
+          throw new \Exception("Too many classes in $path");
+        }
+        else {
+          throw new \Exception("Too few classes in $path");
+        }
+      }
+    }
+
+    return $testClasses;
+  }
+
+  /**
+   * @param array $testClasses
+   * @return array each element is an array with keys:
+   *   - file: string
+   *   - class: string
+   *   - method: string
+   */
+  static function findTestsByPath($paths) {
+    $r = array();
+    $testClasses = self::findTestClasses($paths);
+    foreach ($testClasses as $testFile => $testClass) {
+      $clazz = new \ReflectionClass($testClass);
+      foreach ($clazz->getMethods() as $method) {
+        if (preg_match('/^test/', $method->name)) {
+          $r[] = array(
+            'file' => $testFile,
+            'class' => $testClass,
+            'method' => $method->name
+          );
+        }
+      }
+    }
+    return $r;
+  }
+}
\ No newline at end of file

diff --git a/tools/scripts/jmat b/tools/scripts/jmat
new file mode 100755 (executable)
index 0000000..ae87456
--- /dev/null
+++ b/tools/scripts/jmat
@@ -0,0 +1,105 @@
+#!/bin/bash
+####################################################################
+function show_help() {
+  echo "Jeanine Matthews - Divergent Hunter"
+  echo "summary: Execute test suite in multiple ways and compare results"
+  echo "usage: env [var1=... var2=...] $0 [--bundled-full|--bf] [--bundled-indiv|--bi] [--standalone-full|--sf] [--standalone-indiv|--si] [--compare]"
+  echo "Optional variables:"
+  echo "  - CIVI: Path to Civi installation [$CIVI]"
+  echo "  - PHP: Path to PHP binary [$PHP]"
+  echo "  - PHPUNIT: Path to phpunit binary [$PHPUNIT]"
+  echo "  - TESTCLASS: PHP class name for the test case/suite [$TESTCLASS]"
+  echo "  - TESTPATH: Path for the test file/directory (Note: MUST match TESTCLASS) [$TESTPATH]"
+  echo "  - OUTDIR: Folder to which outputs are written [$OUTDIR]"
+}
+
+function reset_dir() {
+  [ -d "$1" ] && rm -rf "$1"
+  mkdir -p "$1"
+}
+
+####################################################################
+## Env
+export PHP=${PHP:-php}
+export PHPUNIT=${PHPUNIT:-phpunit}
+export TESTCLASS=${TESTCLASS:-api_v3_AllTests}
+export TESTPATH=${TESTPATH:-tests/phpunit/api/v3}
+export CIVI=$(realpath "${CIVI:-.}")
+export OUTDIR=$(realpath "${OUTDIR:-output}")
+
+####################################################################
+## Main
+if [ -z "$1" ];then
+  show_help
+  exit 1
+fi
+
+while [ -n "$1" ]; do
+  OPTION="$1"
+  shift
+
+  case "$OPTION" in
+    --bundled-full|--bf)
+      echo "[[ Prepare $OUTDIR/bundled-full ]]"
+
+      [ -d "packages/PHPUnit.bak" ] && mv "packages/PHPUnit.bak" "packages/PHPUnit"
+      if [ ! -d "$CIVI/packages/PHPUnit" ]; then
+        echo "Missing $CIVI/packages/PHPUnit"
+        exit 2
+      fi
+
+      reset_dir "$OUTDIR/bundled-full"
+
+      pushd "$CIVI/tools"
+        $PHP ./scripts/phpunit --tap --log-json "$OUTDIR/bundled-full/all.json" "$TESTCLASS"
+      popd
+      ;;
+
+    --bundled-indiv|--bi)
+      echo "[[ Prepare $OUTDIR/bundled-indiv ]]"
+
+      [ -d "packages/PHPUnit.bak" ] && mv "packages/PHPUnit.bak" "packages/PHPUnit"
+      if [ ! -d "$CIVI/packages/PHPUnit" ]; then
+        echo "Missing $CIVI/packages/PHPUnit"
+        exit 2
+      fi
+
+      reset_dir "$OUTDIR/bundled-indiv"
+
+      pushd "$CIVI/tools"
+        phpunit-indiv --civi "../$TESTPATH" "$OUTDIR/bundled-indiv"
+      popd
+
+      cat "$OUTDIR"/bundled-indiv/*-*.json > "$OUTDIR/bundled-indiv/all.json"
+      ;;
+
+    --standalone-full|--sf)
+      echo "[[ Prepare $OUTDIR/standalone-full ]]"
+      reset_dir "$OUTDIR/standalone-full"
+
+      pushd "$CIVI"
+        [ -d "packages/PHPUnit" ] && mv "packages/PHPUnit" "packages/PHPUnit.bak"
+        $PHP $(which $PHPUNIT) --tap --log-json "$OUTDIR/standalone-full/all.json" "$TESTPATH"
+        [ -d "packages/PHPUnit.bak" ] && mv "packages/PHPUnit.bak" "packages/PHPUnit"
+      popd
+      ;;
+
+    --standalone-indiv|--si)
+      echo "[[ Prepare $OUTDIR/standalone-indiv ]]"
+      reset_dir "$OUTDIR/standalone-indiv"
+
+      pushd "$CIVI"
+        [ -d "packages/PHPUnit" ] && mv "packages/PHPUnit" "packages/PHPUnit.bak"
+        phpunit-indiv "$TESTPATH" "$OUTDIR/standalone-indiv"
+        [ -d "packages/PHPUnit.bak" ] && mv "packages/PHPUnit.bak" "packages/PHPUnit"
+      popd
+
+      cat "$OUTDIR"/standalone-indiv/*-*.json > "$OUTDIR/standalone-indiv/all.json"
+      ;;
+
+    --compare)
+      echo "[[ Compare all results in $OUTDIR ]]"
+      phpunit-compare "$OUTDIR"/*/all.json
+      ;;
+  esac
+done

diff --git a/tools/scripts/phpunit-antagonist b/tools/scripts/phpunit-antagonist
new file mode 100755 (executable)
index 0000000..996c553
--- /dev/null
+++ b/tools/scripts/phpunit-antagonist
@@ -0,0 +1,4 @@
+#!/usr/bin/env php
+<?php
+require_once dirname(dirname(__DIR__)) . '/tests/phpunit/CiviTest/bootstrap.php';
+\Civi\CiUtil\Command\AntagonistCommand::main($argv);

diff --git a/tools/scripts/phpunit-compare b/tools/scripts/phpunit-compare
new file mode 100755 (executable)
index 0000000..a04c3d7
--- /dev/null
+++ b/tools/scripts/phpunit-compare
@@ -0,0 +1,4 @@
+#!/usr/bin/env php
+<?php
+require_once dirname(dirname(__DIR__)) . '/tests/phpunit/CiviTest/bootstrap.php';
+\Civi\CiUtil\Command\CompareCommand::main($argv);

diff --git a/tools/scripts/phpunit-indiv b/tools/scripts/phpunit-indiv
new file mode 100755 (executable)
index 0000000..6768418
--- /dev/null
+++ b/tools/scripts/phpunit-indiv
@@ -0,0 +1,33 @@
+#!/bin/bash
+PHP=${PHP:-php}
+PHPUNIT=${PHPUNIT:-phpunit}
+MODE=standalone
+
+if [ "$1" == "--civi" ]; then
+  MODE=civi
+  shift
+fi
+
+TESTSUITE="$1"
+shift
+
+OUTDIR="$1"
+shift
+
+if [ -z "$TESTSUITE" -o -z "$OUTDIR" ]; then
+  echo "summary: Executes all tests in a suite (individually)"
+  echo "usage: $0 <test-suite-path> <output-dir>"
+  exit 1
+fi
+
+[ ! -d "$OUTDIR" ] && mkdir -p "$OUTDIR"
+
+#phpunit-ls "$TESTSUITE"
+phpunit-ls "$TESTSUITE" | while read FILE CLASS METHOD ; do
+  if [ "$MODE" == "civi" ]; then
+    $PHP ./scripts/phpunit --tap --log-json "$OUTDIR/$CLASS-$METHOD.json" --filter $METHOD'( with.*)?$' "$CLASS"
+  fi
+  if [ "$MODE" == "standalone" ]; then
+    $PHP $(which $PHPUNIT) --tap --log-json "$OUTDIR/$CLASS-$METHOD.json" --filter $METHOD'( with.*)?$' "$FILE"
+  fi
+done

diff --git a/tools/scripts/phpunit-ls b/tools/scripts/phpunit-ls
new file mode 100755 (executable)
index 0000000..6d6ffc2
--- /dev/null
+++ b/tools/scripts/phpunit-ls
@@ -0,0 +1,5 @@
+#!/usr/bin/env php
+<?php
+require_once dirname(dirname(__DIR__)) . '/tests/phpunit/CiviTest/bootstrap.php';
+error_reporting(E_ALL);
+\Civi\CiUtil\Command\LsCommand::main($argv);
\ No newline at end of file