From fb4ab6230ee38e9e5d12b6a239598ee6fe92c3af Mon Sep 17 00:00:00 2001
From: Tim Otten <totten@civicrm.org>
Date: Thu, 9 Sep 2021 16:26:08 -0700
Subject: [PATCH] TokenProcessor - Allow basic filter/modifier expressions

Before
------

`Hello {foo.bar}`

After
-----

`Hello {foo.bar|upper}` and `Hello {foo.bar|lower}`

Technical Details
-----------------

This only supports tokens that supply values through `civi.token.eval`.  At the time of
this commit, most `{contact.*}` tokens still use `TokenCompatSubscriber::onRender()` to
hack-in their values and won't work until they switch over.

The regex which recognizes the filter is pretty tight (`\w+`).  This can be relaxed
somewhat by a subsequent change, but I'd say such a change has a burden to demonstrate
safety/interoparbility when running in Token-Smarty format.  (e.g.  demonstrate that
matching of open/close symbols works correctly).
---
 Civi/Token/TokenProcessor.php                 | 42 +++++++++++++++++--
 .../phpunit/Civi/Token/TokenProcessorTest.php | 29 +++++++++++++
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/Civi/Token/TokenProcessor.php b/Civi/Token/TokenProcessor.php
index e614ba6408..5411a74fec 100644
--- a/Civi/Token/TokenProcessor.php
+++ b/Civi/Token/TokenProcessor.php
@@ -361,10 +361,13 @@ class TokenProcessor {
     $useSmarty = !empty($row->context['smarty']);
 
     $tokens = $this->rowValues[$row->tokenRow][$message['format']];
-    $getToken = function($m) use ($tokens, $useSmarty) {
+    $getToken = function($m) use ($tokens, $useSmarty, $row) {
       [$full, $entity, $field] = $m;
       if (isset($tokens[$entity][$field])) {
         $v = $tokens[$entity][$field];
+        if (isset($m[3])) {
+          $v = $this->filterTokenValue($v, $m[3], $row);
+        }
         if ($useSmarty) {
           $v = \CRM_Utils_Token::tokenEscapeSmarty($v);
         }
@@ -377,13 +380,44 @@ class TokenProcessor {
     $event->message = $message;
     $event->context = $row->context;
     $event->row = $row;
-    // Regex examples: '{foo.bar}'
-    // Regex counter-examples: '{foobar}', '{foo bar}', '{$foo.bar}', '{$foo.bar|whiz}'
-    $event->string = preg_replace_callback(';\{(\w+)\.(\w+)\};', $getToken, $message['string']);
+    // Regex examples: '{foo.bar}', '{foo.bar|whiz}'
+    // Regex counter-examples: '{foobar}', '{foo bar}', '{$foo.bar}', '{$foo.bar|whiz}', '{foo.bar|whiz{bang}}'
+    // Key observations: Civi tokens MUST have a `.` and MUST NOT have a `$`. Civi filters MUST NOT have `{}`s or `$`s.
+    $tokRegex = '([\w]+)\.([\w:]+)';
+    $filterRegex = '(\w+)';
+    $event->string = preg_replace_callback(";\{$tokRegex(?:\|$filterRegex)?\};", $getToken, $message['string']);
     $this->dispatcher->dispatch('civi.token.render', $event);
     return $event->string;
   }
 
+  /**
+   * Given a token value, run it through any filters.
+   *
+   * @param mixed $value
+   *   Raw token value (e.g. from `$row->tokens['foo']['bar']`).
+   * @param string $filter
+   * @param TokenRow $row
+   *   The current target/row.
+   * @return string
+   * @throws \CRM_Core_Exception
+   */
+  private function filterTokenValue($value, $filter, TokenRow $row) {
+    // KISS demonstration. This should change... e.g. provide a filter-registry or reuse Smarty's registry...
+    switch ($filter) {
+      case NULL:
+        return $value;
+
+      case 'upper':
+        return mb_strtoupper($value);
+
+      case 'lower':
+        return mb_strtolower($value);
+
+      default:
+        throw new \CRM_Core_Exception("Invalid token filter: $filter");
+    }
+  }
+
 }
 
 class TokenRowIterator extends \IteratorIterator {
diff --git a/tests/phpunit/Civi/Token/TokenProcessorTest.php b/tests/phpunit/Civi/Token/TokenProcessorTest.php
index 26e0da7e7f..44f90c13c2 100644
--- a/tests/phpunit/Civi/Token/TokenProcessorTest.php
+++ b/tests/phpunit/Civi/Token/TokenProcessorTest.php
@@ -281,6 +281,35 @@ class TokenProcessorTest extends \CiviUnitTestCase {
     $this->assertEquals(1, $this->counts['onEvalTokens']);
   }
 
+  public function testFilter() {
+    $exampleTokens['foo_bar']['whiz_bang'] = 'Some Text';
+    $exampleMessages = [
+      'This is {foo_bar.whiz_bang}.' => 'This is Some Text.',
+      'This is {foo_bar.whiz_bang|lower}...' => 'This is some text...',
+      'This is {foo_bar.whiz_bang|upper}!' => 'This is SOME TEXT!',
+    ];
+    $expectExampleCount = /* {#msgs} x {smarty:on,off} */ 6;
+    $actualExampleCount = 0;
+
+    foreach ($exampleMessages as $inputMessage => $expectOutput) {
+      foreach ([TRUE, FALSE] as $useSmarty) {
+        $p = new TokenProcessor($this->dispatcher, [
+          'controller' => __CLASS__,
+          'smarty' => $useSmarty,
+        ]);
+        $p->addMessage('example', $inputMessage, 'text/plain');
+        $p->addRow()
+          ->format('text/plain')->tokens($exampleTokens);
+        foreach ($p->evaluate()->getRows() as $key => $row) {
+          $this->assertEquals($expectOutput, $row->render('example'));
+          $actualExampleCount++;
+        }
+      }
+    }
+
+    $this->assertEquals($expectExampleCount, $actualExampleCount);
+  }
+
   public function onListTokens(TokenRegisterEvent $e) {
     $this->counts[__FUNCTION__]++;
     $e->register('custom', [
-- 
2.25.1