(REF) APIv4 ConformanceTest - Split apart into per-entity sub-tests

Overview
--------

This updates a unit-test to make the output easier to understand.

We need more output because the test sporadically fails, and we don't know why.

Before
------

There is a function `testConformance()` which internally
loops through a list of entities.

Whenever the test fails, it aborts testing and reports one failure (for all
entities) without indicating the specific entity which failed.

After
-----

There is a test function `testConformance($entity)` which uses
a data provider.

Whenever the test fails, it will be logged with the entity name.
Testing can resume for additional entities.

Technical Details
-----------------

There are a couple technical distinctions between this revision, the
previous revision, and the comparable APIv3 test:

1. Each test-case returned by the '@dataProvider' has a symbolic name.
   These are easier to skim than numeric names.

2. The list of entities is *not* based on runtime services,
   because that constrains how PHPUnit and Civi lifecycles interact.
   It uses a heuristic/low-tech listing (`getEntitiesLotech()`).
   In the rare case where the low-tech list is wrong, it will complain
   and ask for maintenance.

diff --git a/tests/phpunit/api/v4/Entity/ConformanceTest.php b/tests/phpunit/api/v4/Entity/ConformanceTest.php
index 7c42824ed9ff7df1edf26c4cf8be2a5068ca6b09..1c01956ba30d7ad35a4e74c6b2596342099988be 100644 (file)
--- a/tests/phpunit/api/v4/Entity/ConformanceTest.php
+++ b/tests/phpunit/api/v4/Entity/ConformanceTest.php
@@ -60,43 +60,79 @@ class ConformanceTest extends UnitTestCase {
   /**
    * Get entities to test.
    *
+   * This is the hi-tech list as generated via Civi's runtime services. It
+   * is canonical, but relies on services that may not be available during
+   * early parts of PHPUnit lifecycle.
+   *
    * @return array
    *
    * @throws \API_Exception
    * @throws \Civi\API\Exception\UnauthorizedException
    */
-  public function getEntities() {
-    return Entity::get()->setCheckPermissions(FALSE)->execute()->column('name');
+  public function getEntitiesHitech() {
+    return $this->toDataProviderArray(Entity::get()->setCheckPermissions(FALSE)->execute()->column('name'));
   }
 
   /**
-   * Fixme: This should use getEntities as a dataProvider but that fails for some reason
+   * Get entities to test.
+   *
+   * This is the low-tech list as generated by manual-overrides and direct inspection.
+   * It may be summoned at any time during PHPUnit lifecycle, but it may require
+   * occasional twiddling to give correct results.
+   *
+   * @return array
    */
-  public function testConformance() {
-    $entities = $this->getEntities();
-    $this->assertNotEmpty($entities);
-
-    foreach ($entities as $data) {
-      $entity = $data;
-      $entityClass = 'Civi\Api4\\' . $entity;
-
-      $actions = $this->checkActions($entityClass);
-
-      // Go no further if it's not a CRUD entity
-      if (array_diff(['get', 'create', 'update', 'delete'], array_keys($actions))) {
-        continue;
-      }
-
-      $this->checkFields($entityClass, $entity);
-      $id = $this->checkCreation($entity, $entityClass);
-      $this->checkGet($entityClass, $id, $entity);
-      $this->checkGetCount($entityClass, $id, $entity);
-      $this->checkUpdateFailsFromCreate($entityClass, $id);
-      $this->checkWrongParamType($entityClass);
-      $this->checkDeleteWithNoId($entityClass);
-      $this->checkDeletion($entityClass, $id);
-      $this->checkPostDelete($entityClass, $id, $entity);
+  public function getEntitiesLotech() {
+    $manual['add'] = [];
+    $manual['remove'] = ['CustomValue'];
+
+    $scanned = [];
+    $srcDir = dirname(dirname(dirname(dirname(dirname(__DIR__)))));
+    foreach ((array) glob("$srcDir/Civi/Api4/*.php") as $name) {
+      $scanned[] = preg_replace('/\.php/', '', basename($name));
     }
+
+    $names = array_diff(
+      array_unique(array_merge($scanned, $manual['add'])),
+      $manual['remove']
+    );
+
+    return $this->toDataProviderArray($names);
+  }
+
+  /**
+   * Ensure that "getEntitiesLotech()" (which is the 'dataProvider') is up to date
+   * with "getEntitiesHitech()" (which is a live feed available entities).
+   */
+  public function testEntitiesProvider() {
+    $this->assertEquals($this->getEntitiesHitech(), $this->getEntitiesLotech(), "The lo-tech list of entities does not match the hi-tech list. You probably need to update getEntitiesLotech().");
+  }
+
+  /**
+   * @param string $entity
+   *   Ex: 'Contact'
+   * @dataProvider getEntitiesLotech
+   */
+  public function testConformance($entity) {
+    $entityClass = 'Civi\Api4\\' . $entity;
+
+    $actions = $this->checkActions($entityClass);
+
+    // Go no further if it's not a CRUD entity
+    if (array_diff(['get', 'create', 'update', 'delete'], array_keys($actions))) {
+      $this->markTestSkipped("The entity \"$entity\" does not ");
+      return;
+    }
+
+    $this->checkFields($entityClass, $entity);
+    $id = $this->checkCreation($entity, $entityClass);
+    $this->checkGet($entityClass, $id, $entity);
+    $this->checkGetCount($entityClass, $id, $entity);
+    $this->checkUpdateFailsFromCreate($entityClass, $id);
+    $this->checkWrongParamType($entityClass);
+    $this->checkDeleteWithNoId($entityClass);
+    $this->checkDeletion($entityClass, $id);
+    $this->checkPostDelete($entityClass, $id, $entity);
   }
 
   /**
@@ -270,4 +306,22 @@ class ConformanceTest extends UnitTestCase {
     $this->assertEquals(0, count($getDeletedResult), $errMsg);
   }
 
+  /**
+   * @param array $names
+   *   List of entity names.
+   *   Ex: ['Foo', 'Bar']
+   * @return array
+   *   List of data-provider arguments, one for each entity-name.
+   *   Ex: ['Foo' => ['Foo'], 'Bar' => ['Bar']]
+   */
+  protected function toDataProviderArray($names) {
+    sort($names);
+
+    $result = [];
+    foreach ($names as $name) {
+      $result[$name] = [$name];
+    }
+    return $result;
+  }
+
 }