From 8d4c4d68e06db05a2c64d176ccb83986478762f5 Mon Sep 17 00:00:00 2001
From: Tim Otten <totten@civicrm.org>
Date: Wed, 22 Sep 2021 16:52:47 -0700
Subject: [PATCH] (REF) TokenProcessor::visitTokens() - Increase visibility.
 Refine comments+tests.

---
 Civi/Token/TokenProcessor.php                 | 34 +++++++++++++++----
 .../phpunit/Civi/Token/TokenProcessorTest.php | 19 ++++++-----
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/Civi/Token/TokenProcessor.php b/Civi/Token/TokenProcessor.php
index bb066183a3..cc079ddaa3 100644
--- a/Civi/Token/TokenProcessor.php
+++ b/Civi/Token/TokenProcessor.php
@@ -386,15 +386,37 @@ class TokenProcessor {
     return $event->string;
   }
 
-  private function visitTokens(string $expression, callable $callback): string {
+  /**
+   * Examine a token string and filter each token expression.
+   *
+   * @internal
+   *   This function is only intended for use within civicrm-core. The name/location/callback-signature may change.
+   * @param string $expression
+   *   Ex: 'Hello {foo.bar} and {whiz.bang|filter:"arg"}!'
+   * @param callable $callback
+   *   A function which visits (and substitutes) each token.
+   *   function(?string $fullToken, ?string $entity, ?string $field, ?array $modifier)
+   * @return string
+   */
+  public function visitTokens(string $expression, callable $callback): string {
     // Regex examples: '{foo.bar}', '{foo.bar|whiz}', '{foo.bar|whiz:"bang"}', '{foo.bar|whiz:"bang":"bang"}'
     // Regex counter-examples: '{foobar}', '{foo bar}', '{$foo.bar}', '{$foo.bar|whiz}', '{foo.bar|whiz{bang}}'
     // Key observations: Civi tokens MUST have a `.` and MUST NOT have a `$`. Civi filters MUST NOT have `{}`s or `$`s.
-    $tokRegex = '([\w]+)\.([\w:\.]+)'; /* EX: 'foo.bar' in '{foo.bar|whiz:"bang":"bang"}' */
-    $argRegex = ':[\w": %\-_()\[\]\+/#@!,\.\?]*'; /* EX: ':"bang":"bang"' in '{foo.bar|whiz:"bang":"bang"}' */
-    // Debatable: Maybe relax to this: $argRegex = ':[^{}\n]*'; /* EX: ':"bang":"bang"' in '{foo.bar|whiz:"bang":"bang"}' */
-    $filterRegex = "(\w+(?:$argRegex)?)"; /* EX: 'whiz:"bang"' in '{foo.bar|whiz:"bang"' */
-    return preg_replace_callback(";\{$tokRegex(?:\|$filterRegex)?\};", function($m) use ($callback) {
+
+    static $fullRegex = NULL;
+    if ($fullRegex === NULL) {
+      // The regex is a bit complicated, we so break it down into fragments.
+      // Consider the example '{foo.bar|whiz:"bang":"bang"}'. Each fragment matches the following:
+
+      $tokenRegex = '([\w]+)\.([\w:\.]+)'; /* MATCHES: 'foo.bar' */
+      $filterArgRegex = ':[\w": %\-_()\[\]\+/#@!,\.\?]*'; /* MATCHES: ':"bang":"bang"' */
+      // Key rule of filterArgRegex is to prohibit '{}'s because they may parse ambiguously. So you *might* relax it to:
+      // $filterArgRegex = ':[^{}\n]*'; /* MATCHES: ':"bang":"bang"' */
+      $filterNameRegex = "\w+"; /* MATCHES: 'whiz' */
+      $filterRegex = "\|($filterNameRegex(?:$filterArgRegex)?)"; /* MATCHES: '|whiz:"bang":"bang"' */
+      $fullRegex = ";\{$tokenRegex(?:$filterRegex)?\};";
+    }
+    return preg_replace_callback($fullRegex, function($m) use ($callback) {
       $filterParts = NULL;
       if (isset($m[3])) {
         $filterParts = [];
diff --git a/tests/phpunit/Civi/Token/TokenProcessorTest.php b/tests/phpunit/Civi/Token/TokenProcessorTest.php
index 3c9890c420..1769c7006a 100644
--- a/tests/phpunit/Civi/Token/TokenProcessorTest.php
+++ b/tests/phpunit/Civi/Token/TokenProcessorTest.php
@@ -1,7 +1,6 @@
 <?php
 namespace Civi\Token;
 
-use Civi\Test\Invasive;
 use Civi\Token\Event\TokenRegisterEvent;
 use Civi\Token\Event\TokenValueEvent;
 use Symfony\Component\EventDispatcher\EventDispatcher;
@@ -45,19 +44,23 @@ class TokenProcessorTest extends \CiviUnitTestCase {
       '{foo.bar}' => ['foo', 'bar', NULL],
       '{foo.bar|whiz}' => ['foo', 'bar', ['whiz']],
       '{foo.bar|whiz:"bang"}' => ['foo', 'bar', ['whiz', 'bang']],
-      '{love.shack|place:"bang":"b@ng, on +he/([do0r])?!"}' => ['love', 'shack', ['place', 'bang', 'b@ng, on +he/([do0r])?!']],
+      '{FoO.bAr|whiz:"bang"}' => ['FoO', 'bAr', ['whiz', 'bang']],
+      '{oo_f.ra_b|b_52:"bang":"b@ng, on +he/([do0r])?!"}' => ['oo_f', 'ra_b', ['b_52', 'bang', 'b@ng, on +he/([do0r])?!']],
+      '{foo.bar.whiz}' => ['foo', 'bar.whiz', NULL],
+      '{foo.bar.whiz|bang}' => ['foo', 'bar.whiz', ['bang']],
+      '{foo.bar:label}' => ['foo', 'bar:label', NULL],
+      '{foo.bar:label|truncate:"10"}' => ['foo', 'bar:label', ['truncate', '10']],
     ];
     foreach ($examples as $input => $expected) {
       array_unshift($expected, $input);
       $log = [];
-      Invasive::call([$p, 'visitTokens'], [
-        $input,
-        function (?string $fullToken, ?string $entity, ?string $field, ?array $modifier) use (&$log) {
-          $log[] = [$fullToken, $entity, $field, $modifier];
-        },
-      ]);
+      $filtered = $p->visitTokens($input, function (?string $fullToken, ?string $entity, ?string $field, ?array $modifier) use (&$log) {
+        $log[] = [$fullToken, $entity, $field, $modifier];
+        return 'Replaced!';
+      });
       $this->assertEquals(1, count($log), "Should receive one callback on expression: $input");
       $this->assertEquals($expected, $log[0]);
+      $this->assertEquals('Replaced!', $filtered);
     }
   }
 
-- 
2.25.1