From 8adcd0735bc955721841ac7f04a3e3bc13ebb252 Mon Sep 17 00:00:00 2001
From: Tim Otten <totten@civicrm.org>
Date: Thu, 23 Jul 2015 00:11:45 -0700
Subject: [PATCH] CRM-13244 - Civi\Token - Add new class for processing
 messages with tokens.

---
 Civi/Token/Event/TokenEvent.php               |  25 ++
 Civi/Token/Event/TokenRegisterEvent.php       |  70 +++++
 Civi/Token/Event/TokenRenderEvent.php         |  44 +++
 Civi/Token/Event/TokenValueEvent.php          |  45 +++
 Civi/Token/Events.php                         |  32 +++
 Civi/Token/TokenException.php                 |   6 +
 Civi/Token/TokenProcessor.php                 | 251 +++++++++++++++++
 Civi/Token/TokenRow.php                       | 256 ++++++++++++++++++
 .../phpunit/Civi/Token/TokenProcessorTest.php | 169 ++++++++++++
 9 files changed, 898 insertions(+)
 create mode 100644 Civi/Token/Event/TokenEvent.php
 create mode 100644 Civi/Token/Event/TokenRegisterEvent.php
 create mode 100644 Civi/Token/Event/TokenRenderEvent.php
 create mode 100644 Civi/Token/Event/TokenValueEvent.php
 create mode 100644 Civi/Token/Events.php
 create mode 100644 Civi/Token/TokenException.php
 create mode 100644 Civi/Token/TokenProcessor.php
 create mode 100644 Civi/Token/TokenRow.php
 create mode 100644 tests/phpunit/Civi/Token/TokenProcessorTest.php

diff --git a/Civi/Token/Event/TokenEvent.php b/Civi/Token/Event/TokenEvent.php
new file mode 100644
index 0000000000..10944eb9a4
--- /dev/null
+++ b/Civi/Token/Event/TokenEvent.php
@@ -0,0 +1,25 @@
+<?php
+namespace Civi\Token\Event;
+
+use Symfony\Component\EventDispatcher\Event;
+
+/**
+ * Class TokenListEvent
+ * @package Civi\Token\Event
+ */
+class TokenEvent extends Event {
+
+  protected $tokenProcessor;
+
+  public function __construct($tokenProcessor) {
+    $this->tokenProcessor = $tokenProcessor;
+  }
+
+  /**
+   * @return \Civi\Token\TokenProcessor
+   */
+  public function getTokenProcessor() {
+    return $this->tokenProcessor;
+  }
+
+}
diff --git a/Civi/Token/Event/TokenRegisterEvent.php b/Civi/Token/Event/TokenRegisterEvent.php
new file mode 100644
index 0000000000..067f7e49cc
--- /dev/null
+++ b/Civi/Token/Event/TokenRegisterEvent.php
@@ -0,0 +1,70 @@
+<?php
+namespace Civi\Token\Event;
+
+/**
+ * Class TokenRegisterEvent
+ * @package Civi\Token\Event
+ *
+ * The TokenRegisterEvent is fired when constructing a list of available
+ * tokens. Listeners may register by specifying the entity/field/label for the token.
+ *
+ * @code
+ * $ev->entity('profile')
+ *    ->register('viewUrl', ts('Default Profile URL (View Mode)')
+ *    ->register('editUrl', ts('Default Profile URL (Edit Mode)');
+ * $ev->register(array(
+ *   'entity' => 'profile',
+ *   'field' => 'viewUrl',
+ *   'label' => ts('Default Profile URL (View Mode)'),
+ * ));
+ * @endcode
+ */
+class TokenRegisterEvent extends TokenEvent {
+
+  /**
+   * Default values to put in new registrations.
+   *
+   * @var array
+   */
+  protected $defaults;
+
+  public function __construct($tokenProcessor, $defaults) {
+    parent::__construct($tokenProcessor);
+    $this->defaults = $defaults;
+  }
+
+  /**
+   * Set the default entity name.
+   *
+   * @param string $entity
+   * @return TokenRegisterEvent
+   */
+  public function entity($entity) {
+    $defaults = $this->defaults;
+    $defaults['entity'] = $entity;
+    return new TokenRegisterEvent($this->tokenProcessor, $defaults);
+  }
+
+  /**
+   * Register a new token.
+   *
+   * @param array|string $paramsOrField
+   * @param NULL|string $label
+   * @return $this
+   */
+  public function register($paramsOrField, $label = NULL) {
+    if (is_array($paramsOrField)) {
+      $params = $paramsOrField;
+    }
+    else {
+      $params = array(
+        'field' => $paramsOrField,
+        'label' => $label,
+      );
+    }
+    $params = array_merge($this->defaults, $params);
+    $this->tokenProcessor->addToken($params);
+    return $this;
+  }
+
+}
diff --git a/Civi/Token/Event/TokenRenderEvent.php b/Civi/Token/Event/TokenRenderEvent.php
new file mode 100644
index 0000000000..3fda8442eb
--- /dev/null
+++ b/Civi/Token/Event/TokenRenderEvent.php
@@ -0,0 +1,44 @@
+<?php
+namespace Civi\Token\Event;
+
+/**
+ * Class TokenRenderEvent
+ * @package Civi\Token\Event
+ *
+ * A TokenRenderEvent is fired after the TokenProcessor has rendered
+ * a message.
+ *
+ * The render event may be used for post-processing the text, but
+ * it's very difficult to do substantive work in a secure, robust
+ * way within this event. The event primarily exists to facilitate
+ * a transition of some legacy code.
+ */
+class TokenRenderEvent extends TokenEvent {
+
+  /**
+   * @var array|\ArrayAccess
+   */
+  public $context;
+
+  /**
+   * @var array|\ArrayAccess
+   *
+   * The original message template.
+   */
+  public $message;
+
+  /**
+   * @var \Civi\Token\TokenRow
+   *
+   * The record for which we're generating date
+   */
+  public $row;
+
+  /**
+   * @var string
+   *
+   * The rendered string, with tokens replaced.
+   */
+  public $string;
+
+}
diff --git a/Civi/Token/Event/TokenValueEvent.php b/Civi/Token/Event/TokenValueEvent.php
new file mode 100644
index 0000000000..5201ecdd65
--- /dev/null
+++ b/Civi/Token/Event/TokenValueEvent.php
@@ -0,0 +1,45 @@
+<?php
+namespace Civi\Token\Event;
+
+/**
+ * Class TokenValueEvent
+ * @package Civi\Token\Event
+ *
+ * A TokenValueEvent is fired to convert raw query data into mergeable
+ * tokens. For example:
+ *
+ * @code
+ * $event = new TokenValueEvent($myContext, 'text/html', array(
+ *   array('contact_id' => 123),
+ *   array('contact_id' => 456),
+ * ));
+ *
+ * // Compute tokens one row at a time.
+ * foreach ($event->getRows() as $row) {
+ *   $row->setTokens('contact', array(
+ *     'profileUrl' => CRM_Utils_System::url('civicrm/profile/view', 'reset=1&gid=456&id=' . $row['contact_id']'),
+ *   ));
+ * }
+ *
+ * // Compute tokens with a bulk lookup.
+ * $ids = implode(',', array_filter(CRM_Utils_Array::collect('contact_id', $event->getRows()), 'is_numeric'));
+ * $dao = CRM_Core_DAO::executeQuery("SELECT contact_id, foo, bar FROM foobar WHERE contact_id in ($ids)");
+ * while ($dao->fetch) {
+ *   $row->setTokens('oddball', array(
+ *     'foo' => $dao->foo,
+ *     'bar' => $dao->bar,
+ * ));
+ * }
+ * @encode
+ *
+ */
+class TokenValueEvent extends TokenEvent {
+
+  /**
+   * @return \Traversable<TokenRow>
+   */
+  public function getRows() {
+    return $this->tokenProcessor->getRows();
+  }
+
+}
diff --git a/Civi/Token/Events.php b/Civi/Token/Events.php
new file mode 100644
index 0000000000..7c7869ef09
--- /dev/null
+++ b/Civi/Token/Events.php
@@ -0,0 +1,32 @@
+<?php
+
+namespace Civi\Token;
+
+class Events {
+  /**
+   * Create a list of supported tokens.
+   *
+   * @see \Civi\Token\Event\TokenRegisterEvent
+   */
+  const TOKEN_REGISTER = 'civi.token.list';
+
+  /**
+   * Create a list of supported tokens.
+   *
+   * @see \Civi\Token\Event\TokenValueEvent
+   */
+  const TOKEN_EVALUATE = 'civi.token.eval';
+
+  /**
+   * Perform post-processing on a rendered message.
+   *
+   * WARNING: It is difficult to develop robust,
+   * secure code using this stage. However, we need
+   * to support it during a transitional period
+   * while the token logic is reorganized.
+   *
+   * @see \Civi\Token\Event\TokenRenderEvent
+   */
+  const TOKEN_RENDER = 'civi.token.render';
+
+}
diff --git a/Civi/Token/TokenException.php b/Civi/Token/TokenException.php
new file mode 100644
index 0000000000..106c762128
--- /dev/null
+++ b/Civi/Token/TokenException.php
@@ -0,0 +1,6 @@
+<?php
+namespace Civi\Token;
+
+class TokenException extends \CRM_Core_Exception {
+
+}
diff --git a/Civi/Token/TokenProcessor.php b/Civi/Token/TokenProcessor.php
new file mode 100644
index 0000000000..b373a80686
--- /dev/null
+++ b/Civi/Token/TokenProcessor.php
@@ -0,0 +1,251 @@
+<?php
+namespace Civi\Token;
+
+use Civi\Token\Event\TokenRegisterEvent;
+use Civi\Token\Event\TokenRenderEvent;
+use Civi\Token\Event\TokenValueEvent;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Traversable;
+
+class TokenProcessor {
+
+  /**
+   * @var array
+   *   Description of the context in which the tokens are being processed.
+   *   Ex: Array('class'=>'CRM_Core_BAO_ActionSchedule', 'schedule' => $dao, 'mapping' => $dao).
+   *   Ex: Array('class'=>'CRM_Mailing_BAO_MailingJob', 'mailing' => $dao).
+   *
+   * For lack of a better place, here's a list of known/intended context values:
+   *
+   *   - controller: string, the class which is managing the mail-merge.
+   *   - smarty: bool, whether to enable smarty support.
+   *   - contactId: int, the main person/org discussed in the message.
+   *   - contact: array, the main person/org discussed in the message.
+   *     (Optional for performance tweaking; if omitted, will load
+   *     automatically from contactId.)
+   *   - actionSchedule: DAO, the rule which triggered the mailing
+   *     [for CRM_Core_BAO_ActionScheduler].
+   */
+  public $context;
+
+  /**
+   * @var EventDispatcherInterface
+   */
+  protected $dispatcher;
+
+  /**
+   * @var array
+   *   Each message is an array with keys:
+   *    - string: Unprocessed message (eg "Hello, {display_name}.").
+   *    - format: Media type (eg "text/plain").
+   *    - tokens: List of tokens which are actually used in this message.
+   */
+  protected $messages;
+
+  /**
+   * DO NOT access field this directly. Use TokenRow. This is
+   * marked as public only to benefit TokenRow.
+   *
+   * @var array
+   *   Array(int $pos => array $keyValues);
+   */
+  public $rowContexts;
+
+  /**
+   * DO NOT access field this directly. Use TokenRow. This is
+   * marked as public only to benefit TokenRow.
+   *
+   * @var array
+   *   Ex: $rowValues[$rowPos][$format][$entity][$field] = 'something';
+   *    Ex: $rowValues[3]['text/plain']['contact']['display_name'] = 'something';
+   */
+  public $rowValues;
+
+  /**
+   * A list of available tokens
+   * @var array
+   *   Array(string $dottedName => array('entity'=>string, 'field'=>string, 'label'=>string)).
+   */
+  protected $tokens = NULL;
+
+  protected $next = 0;
+
+  /**
+   * @param EventDispatcherInterface $dispatcher
+   * @param array $context
+   */
+  public function __construct($dispatcher, $context) {
+    $this->dispatcher = $dispatcher;
+    $this->context = $context;
+  }
+
+  /**
+   * Register a string for which we'll need to merge in tokens.
+   *
+   * @param string $name
+   *   Ex: 'subject', 'body_html'.
+   * @param string $value
+   *   Ex: '<p>Hello {contact.name}</p>'.
+   * @param string $format
+   *   Ex: 'text/html'.
+   * @return $this
+   */
+  public function addMessage($name, $value, $format) {
+    $this->messages[$name] = array(
+      'string' => $value,
+      'format' => $format,
+      'tokens' => \CRM_Utils_Token::getTokens($value),
+    );
+    return $this;
+  }
+
+  /**
+   * Add a row of data.
+   *
+   * @return TokenRow
+   */
+  public function addRow() {
+    $key = $this->next++;
+    $this->rowContexts[$key] = array();
+    $this->rowValues[$key] = array(
+      'text/plain' => array(),
+      'text/html' => array(),
+    );
+
+    return new TokenRow($this, $key);
+  }
+
+  /**
+   * @param array $params
+   *   Array with keys:
+   *    - entity: string, e.g. "profile".
+   *    - field: string, e.g. "viewUrl".
+   *    - label: string, e.g. "Default Profile URL (View Mode)".
+   * @return $this
+   */
+  public function addToken($params) {
+    $key = $params['entity'] . '.' . $params['field'];
+    $this->tokens[$key] = $params;
+    return $this;
+  }
+
+  /**
+   * @param string $name
+   * @return array
+   *   Keys:
+   *    - string: Unprocessed message (eg "Hello, {display_name}.").
+   *    - format: Media type (eg "text/plain").
+   */
+  public function getMessage($name) {
+    return $this->messages[$name];
+  }
+
+  /**
+   * Get a list of all tokens used in registered messages.
+   *
+   * @return array
+   */
+  public function getMessageTokens() {
+    $tokens = array();
+    foreach ($this->messages as $message) {
+      $tokens = \CRM_Utils_Array::crmArrayMerge($tokens, $message['tokens']);
+    }
+    foreach (array_keys($tokens) as $e) {
+      $tokens[$e] = array_unique($tokens[$e]);
+      sort($tokens[$e]);
+    }
+    return $tokens;
+  }
+
+  public function getRow($key) {
+    return new TokenRow($this, $key);
+  }
+
+  /**
+   * @return \Traversable<TokenRow>
+   */
+  public function getRows() {
+    return new TokenRowIterator($this, new \ArrayIterator($this->rowContexts));
+  }
+
+  /**
+   * Get the list of available tokens.
+   *
+   * @return array
+   *   Ex: $tokens['event'] = array('location', 'start_date', 'end_date').
+   */
+  public function getTokens() {
+    if ($this->tokens === NULL) {
+      $this->tokens = array();
+      $event = new TokenRegisterEvent($this, array('entity' => 'undefined'));
+      $this->dispatcher->dispatch(Events::TOKEN_REGISTER, $event);
+    }
+    return $this->tokens;
+  }
+
+  /**
+   * Compute and store token values.
+   */
+  public function evaluate() {
+    $event = new TokenValueEvent($this);
+    $this->dispatcher->dispatch(Events::TOKEN_EVALUATE, $event);
+    return $this;
+  }
+
+  /**
+   * Render a message.
+   *
+   * @param string $name
+   *   The name previously registered with addMessage().
+   * @param TokenRow|int $row
+   *   The object or ID for the row previously registered with addRow().
+   * @return string
+   *   Fully rendered message, with tokens merged.
+   */
+  public function render($name, $row) {
+    if (!is_object($row)) {
+      $row = $this->getRow($row);
+    }
+
+    $message = $this->getMessage($name);
+    $row->fill($message['format']);
+    $useSmarty = !empty($row->context['smarty']);
+
+    // FIXME preg_callback.
+    $tokens = $this->rowValues[$row->tokenRow][$message['format']];
+    $flatTokens = array();
+    \CRM_Utils_Array::flatten($tokens, $flatTokens, '', '.');
+    $filteredTokens = array();
+    foreach ($flatTokens as $k => $v) {
+      $filteredTokens['{' . $k . '}'] = ($useSmarty ? \CRM_Utils_Token::tokenEscapeSmarty($v) : $v);
+    }
+
+    $event = new TokenRenderEvent($this);
+    $event->message = $message;
+    $event->context = $row->context;
+    $event->row = $row;
+    $event->string = strtr($message['string'], $filteredTokens);
+    $this->dispatcher->dispatch(Events::TOKEN_RENDER, $event);
+    return $event->string;
+  }
+
+}
+
+class TokenRowIterator extends \IteratorIterator {
+
+  protected $tokenProcessor;
+
+  /**
+   * @param TokenProcessor $tokenProcessor
+   * @param Traversable $iterator
+   */
+  public function __construct(TokenProcessor $tokenProcessor, Traversable $iterator) {
+    parent::__construct($iterator); // TODO: Change the autogenerated stub
+    $this->tokenProcessor = $tokenProcessor;
+  }
+
+  public function current() {
+    return new TokenRow($this->tokenProcessor, parent::key());
+  }
+
+}
diff --git a/Civi/Token/TokenRow.php b/Civi/Token/TokenRow.php
new file mode 100644
index 0000000000..f05c2f7909
--- /dev/null
+++ b/Civi/Token/TokenRow.php
@@ -0,0 +1,256 @@
+<?php
+namespace Civi\Token;
+use Civi\Token\Event\TokenRenderEvent;
+
+/**
+ * Class TokenRow
+ * @package Civi\Token
+ *
+ * A TokenRow is a helper providing simplified access to the
+ * TokenProcessor.
+ *
+ * A TokenRow combines two elements:
+ *   - context: This is backend data provided by the controller.
+ *   - tokens: This is frontend data that can be mail-merged.
+ *
+ * The context and tokens can be accessed using either methods
+ * or attributes. The methods are appropriate for updates
+ * (and generally accept a mix of arrays), and the attributes
+ * are appropriate for reads.
+ *
+ * To update the context or the tokens, use the methods.
+ * Note that the methods are fairly flexible about accepting
+ * single values or arrays. If given an array, the values
+ * will be merged recursively.
+ *
+ * @code
+ * $row
+ *   ->context('contact_id', 123)
+ *   ->context(array('contact_id' => 123))
+ *   ->tokens('profile', array('viewUrl' => 'http://example.com'))
+ *   ->tokens('profile', 'viewUrl, 'http://example.com');
+ *
+ * echo $row->context['contact_id'];
+ * echo $row->tokens['profile']['viewUrl'];
+ *
+ * $row->tokens('profile', array(
+ *   'viewUrl' => 'http://example.com/view/' . urlencode($row->context['contact_id'];
+ * ));
+ * @endcode
+ */
+class TokenRow {
+
+  /**
+   * @var TokenProcessor
+   */
+  public $tokenProcessor;
+
+  public $tokenRow;
+
+  public $format;
+
+  /**
+   * @var array|ArrayAccess
+   *   List of token values.
+   *   Ex: array('contact' => array('display_name' => 'Alice')).
+   */
+  public $tokens;
+
+  /**
+   * @var array|ArrayAccess
+   *   List of context values.
+   *   Ex: array('controller' => 'CRM_Foo_Bar').
+   */
+  public $context;
+
+  public function __construct(TokenProcessor $tokenProcessor, $key) {
+    $this->tokenProcessor = $tokenProcessor;
+    $this->tokenRow = $key;
+    $this->format('text/plain'); // Set a default.
+    $this->context = new TokenRowContext($tokenProcessor, $key);
+  }
+
+  /**
+   * @param string $format
+   * @return $this
+   */
+  public function format($format) {
+    $this->format = $format;
+    $this->tokens = &$this->tokenProcessor->rowValues[$this->tokenRow][$format];
+    return $this;
+  }
+
+  /**
+   * Update the value of a context element.
+   *
+   * @param string|array $a
+   * @param mixed $b
+   * @return $this
+   */
+  public function context($a = NULL, $b = NULL) {
+    if (is_array($a)) {
+      \CRM_Utils_Array::extend($this->tokenProcessor->rowContexts[$this->tokenRow], $a);
+    }
+    elseif (is_array($b)) {
+      \CRM_Utils_Array::extend($this->tokenProcessor->rowContexts[$this->tokenRow][$a], $b);
+    }
+    else {
+      $this->tokenProcessor->rowContexts[$this->tokenRow][$a] = $b;
+    }
+    return $this;
+  }
+
+  /**
+   * Update the value of a token.
+   *
+   * @param string|array $a
+   * @param string|array $b
+   * @param mixed $c
+   * @return $this
+   */
+  public function tokens($a = NULL, $b = NULL, $c = NULL) {
+    if (is_array($a)) {
+      \CRM_Utils_Array::extend($this->tokens, $a);
+    }
+    elseif (is_array($b)) {
+      \CRM_Utils_Array::extend($this->tokens[$a], $b);
+    }
+    elseif (is_array($c)) {
+      \CRM_Utils_Array::extend($this->tokens[$a][$b], $c);
+    }
+    elseif ($c === NULL) {
+      $this->tokens[$a] = $b;
+    }
+    else {
+      $this->tokens[$a][$b] = $c;
+    }
+    return $this;
+  }
+
+  /**
+   * Auto-convert between different formats
+   */
+  public function fill($format = NULL) {
+    if ($format === NULL) {
+      $format = $this->format;
+    }
+
+    if (!isset($this->tokenProcessor->rowValues[$this->tokenRow]['text/html'])) {
+      $this->tokenProcessor->rowValues[$this->tokenRow]['text/html'] = array();
+    }
+    if (!isset($this->tokenProcessor->rowValues[$this->tokenRow]['text/plain'])) {
+      $this->tokenProcessor->rowValues[$this->tokenRow]['text/plain'] = array();
+    }
+
+    $htmlTokens = &$this->tokenProcessor->rowValues[$this->tokenRow]['text/html'];
+    $textTokens = &$this->tokenProcessor->rowValues[$this->tokenRow]['text/plain'];
+
+    switch ($format) {
+      case 'text/html':
+        // Plain => HTML.
+        foreach ($textTokens as $entity => $values) {
+          foreach ($values as $field => $value) {
+            if (!isset($htmlTokens[$entity][$field])) {
+              $htmlTokens[$entity][$field] = htmlentities($value);
+            }
+          }
+        }
+        break;
+
+      case 'text/plain':
+        // HTML => Plain.
+        foreach ($htmlTokens as $entity => $values) {
+          foreach ($values as $field => $value) {
+            if (!isset($textTokens[$entity][$field])) {
+              $textTokens[$entity][$field] = html_entity_decode(strip_tags($value));
+            }
+          }
+        }
+        break;
+
+      default:
+        throw new \RuntimeException("Invalid format");
+    }
+
+    return $this;
+  }
+
+  /**
+   * Render a message.
+   *
+   * @param string $name
+   *   The name previously registered with TokenProcessor::addMessage.
+   * @return string
+   *   Fully rendered message, with tokens merged.
+   */
+  public function render($name) {
+    return $this->tokenProcessor->render($name, $this);
+  }
+
+}
+
+/**
+ * Class TokenRowContext
+ * @package Civi\Token
+ *
+ * Combine the row-context and general-context into a single array-like facade.
+ */
+class TokenRowContext implements \ArrayAccess, \IteratorAggregate, \Countable {
+
+  /**
+   * @var TokenProcessor
+   */
+  protected $tokenProcessor;
+
+  protected $tokenRow;
+
+  /**
+   * @param $tokenProcessor
+   * @param $tokenRow
+   */
+  public function __construct($tokenProcessor, $tokenRow) {
+    $this->tokenProcessor = $tokenProcessor;
+    $this->tokenRow = $tokenRow;
+  }
+
+  public function offsetExists($offset) {
+    return
+      isset($this->tokenProcessor->rowContexts[$this->tokenRow][$offset])
+      || isset($this->tokenProcessor->context[$offset]);
+  }
+
+  public function &offsetGet($offset) {
+    if (isset($this->tokenProcessor->rowContexts[$this->tokenRow][$offset])) {
+      return $this->tokenProcessor->rowContexts[$this->tokenRow][$offset];
+    }
+    if (isset($this->tokenProcessor->context[$offset])) {
+      return $this->tokenProcessor->context[$offset];
+    }
+    $val = NULL;
+    return $val;
+  }
+
+  public function offsetSet($offset, $value) {
+    $this->tokenProcessor->rowContexts[$this->tokenRow][$offset] = $value;
+  }
+
+  public function offsetUnset($offset) {
+    unset($this->tokenProcessor->rowContexts[$this->tokenRow][$offset]);
+  }
+
+  public function getIterator() {
+    return new \ArrayIterator($this->createMergedArray());
+  }
+
+  public function count() {
+    return count($this->createMergedArray());
+  }
+
+  protected function createMergedArray() {
+    return array_merge(
+      $this->tokenProcessor->rowContexts[$this->tokenRow],
+      $this->tokenProcessor->context
+    );
+  }
+
+}
diff --git a/tests/phpunit/Civi/Token/TokenProcessorTest.php b/tests/phpunit/Civi/Token/TokenProcessorTest.php
new file mode 100644
index 0000000000..6ddf64d850
--- /dev/null
+++ b/tests/phpunit/Civi/Token/TokenProcessorTest.php
@@ -0,0 +1,169 @@
+<?php
+namespace Civi\Token;
+
+require_once 'CiviTest/CiviUnitTestCase.php';
+
+use Civi\Token\Event\TokenRegisterEvent;
+use Civi\Token\Event\TokenValueEvent;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+class TokenProcessorTest extends \CiviUnitTestCase {
+
+  /**
+   * @var EventDispatcher
+   */
+  protected $dispatcher;
+
+  /**
+   * @var array
+   *   Array(string $funcName => int $invocationCount).
+   */
+  protected $counts;
+
+  protected function setUp() {
+    $this->useTransaction(TRUE);
+    parent::setUp();
+    $this->dispatcher = new EventDispatcher();
+    $this->dispatcher->addListener(Events::TOKEN_REGISTER, array($this, 'onListTokens'));
+    $this->dispatcher->addListener(Events::TOKEN_EVALUATE, array($this, 'onEvalTokens'));
+    $this->counts = array(
+      'onListTokens' => 0,
+      'onEvalTokens' => 0,
+    );
+  }
+
+  /**
+   * Check that the TokenRow helper can correctly read/update context
+   * values.
+   */
+  public function testRowContext() {
+    $p = new TokenProcessor($this->dispatcher, array(
+      'controller' => __CLASS__,
+      'omega' => '99',
+    ));
+    $createdRow = $p->addRow()
+      ->context('one', 1)
+      ->context('two', array(2 => 3))
+      ->context(array(
+        'two' => array(4 => 5),
+        'three' => array(6 => 7),
+        'omega' => '98',
+      ));
+    $gotRow = $p->getRow(0);
+    foreach (array($createdRow, $gotRow) as $row) {
+      $this->assertEquals(1, $row->context['one']);
+      $this->assertEquals(3, $row->context['two'][2]);
+      $this->assertEquals(5, $row->context['two'][4]);
+      $this->assertEquals(7, $row->context['three'][6]);
+      $this->assertEquals(98, $row->context['omega']);
+      $this->assertEquals(__CLASS__, $row->context['controller']);
+    }
+  }
+
+  /**
+   * Check that the TokenRow helper can correctly read/update token
+   * values.
+   */
+  public function testRowTokens() {
+    $p = new TokenProcessor($this->dispatcher, array(
+      'controller' => __CLASS__,
+    ));
+    $createdRow = $p->addRow()
+      ->tokens('one', 1)
+      ->tokens('two', array(2 => 3))
+      ->tokens(array(
+        'two' => array(4 => 5),
+        'three' => array(6 => 7),
+      ))
+      ->tokens('four', 8, 9);
+    $gotRow = $p->getRow(0);
+    foreach (array($createdRow, $gotRow) as $row) {
+      $this->assertEquals(1, $row->tokens['one']);
+      $this->assertEquals(3, $row->tokens['two'][2]);
+      $this->assertEquals(5, $row->tokens['two'][4]);
+      $this->assertEquals(7, $row->tokens['three'][6]);
+      $this->assertEquals(9, $row->tokens['four'][8]);
+    }
+  }
+
+  public function testGetMessageTokens() {
+    $p = new TokenProcessor($this->dispatcher, array(
+      'controller' => __CLASS__,
+    ));
+    $p->addMessage('greeting_html', 'Good morning, <p>{contact.display_name}</p>. {custom.foobar}!', 'text/html');
+    $p->addMessage('greeting_text', 'Good morning, {contact.display_name}. {custom.whizbang}, {contact.first_name}!', 'text/plain');
+    $expected = array(
+      'contact' => array('display_name', 'first_name'),
+      'custom' => array('foobar', 'whizbang'),
+    );
+    $this->assertEquals($expected, $p->getMessageTokens());
+  }
+
+  /**
+   * Perform a full mail-merge, substituting multiple tokens for multiple
+   * contacts in multiple messages.
+   */
+  public function testFull() {
+    $p = new TokenProcessor($this->dispatcher, array(
+      'controller' => __CLASS__,
+    ));
+    $p->addMessage('greeting_html', 'Good morning, <p>{contact.display_name}</p>. {custom.foobar} Bye!', 'text/html');
+    $p->addMessage('greeting_text', 'Good morning, {contact.display_name}. {custom.foobar} Bye!', 'text/plain');
+    $p->addRow()
+      ->context(array('contact_id' => 123))
+      ->format('text/plain')->tokens(array(
+        'contact' => array('display_name' => 'What'),
+      ));
+    $p->addRow()
+      ->context(array('contact_id' => 4))
+      ->format('text/plain')->tokens(array(
+        'contact' => array('display_name' => 'Who'),
+      ));
+    $p->addRow()
+      ->context(array('contact_id' => 10))
+      ->format('text/plain')->tokens(array(
+        'contact' => array('display_name' => 'Darth Vader'),
+      ));
+
+    $expectHtml = array(
+      0 => 'Good morning, <p>What</p>. #0123 is a good number. Trickster {contact.display_name}. Bye!',
+      1 => 'Good morning, <p>Who</p>. #0004 is a good number. Trickster {contact.display_name}. Bye!',
+      2 => 'Good morning, <p>Darth Vader</p>. #0010 is a good number. Trickster {contact.display_name}. Bye!',
+    );
+
+    $expectText = array(
+      0 => 'Good morning, What. #0123 is a good number. Trickster {contact.display_name}. Bye!' ,
+      1 => 'Good morning, Who. #0004 is a good number. Trickster {contact.display_name}. Bye!',
+      2 => 'Good morning, Darth Vader. #0010 is a good number. Trickster {contact.display_name}. Bye!',
+    );
+
+    $rowCount = 0;
+    foreach ($p->evaluate()->getRows() as $key => $row) {
+      /** @var TokenRow */
+      $this->assertTrue($row instanceof TokenRow);
+      $this->assertEquals($expectHtml[$key], $row->render('greeting_html'));
+      $this->assertEquals($expectText[$key], $row->render('greeting_text'));
+      $rowCount++;
+    }
+    $this->assertEquals(3, $rowCount);
+    $this->assertEquals(0, $this->counts['onListTokens']); // This may change in the future.
+    $this->assertEquals(1, $this->counts['onEvalTokens']);
+  }
+
+  public function onListTokens(TokenRegisterEvent $e) {
+    $this->counts[__FUNCTION__]++;
+    $e->register('custom', array(
+      'foobar' => 'A special message about foobar',
+    ));
+  }
+
+  public function onEvalTokens(TokenValueEvent $e) {
+    $this->counts[__FUNCTION__]++;
+    foreach ($e->getRows() as $row) {
+      /** @var TokenRow $row */
+      $row->format('text/html');
+      $row->tokens['custom']['foobar'] = sprintf("#%04d is a good number. Trickster {contact.display_name}.", $row->context['contact_id']);
+    }
+  }
+
+}
-- 
2.25.1