From 79af442ebec5e9fa963efd233f61b1f770cda875 Mon Sep 17 00:00:00 2001
From: Eileen McNaughton <emcnaughton@wikimedia.org>
Date: Wed, 30 Aug 2023 13:31:07 +1200
Subject: [PATCH] Convert html to text, keeping line breaks, when using
 html-based tokens

---
 Civi/Token/TokenRow.php                          | 11 +++++++++--
 tests/phpunit/CRM/Utils/TokenConsistencyTest.php |  9 ++++++++-
 tests/phpunit/Civi/Token/TokenProcessorTest.php  |  8 +++++---
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/Civi/Token/TokenRow.php b/Civi/Token/TokenRow.php
index 0ef5b067e3..53fa7f0114 100644
--- a/Civi/Token/TokenRow.php
+++ b/Civi/Token/TokenRow.php
@@ -188,11 +188,17 @@ class TokenRow {
       'id' => $entityID,
     ]);
     $fieldValue = \CRM_Utils_Array::value($customFieldName, $record, '');
-
+    $originalValue = $fieldValue;
     // format the raw custom field value into proper display value
     if (isset($fieldValue)) {
       $fieldValue = (string) \CRM_Core_BAO_CustomField::displayValue($fieldValue, $customFieldID);
     }
+    // This is a bit of a clumsy wy of detecting a link field but if you look into the displayValue
+    // function you will understand.... By assigning the url as a plain token the text version can
+    // use it as plain text (not html re-converted which kinda works but not in subject lines)
+    if (is_string($fieldValue) && is_string($originalValue) && strpos($fieldValue, '<a href') !== FALSE && strpos($originalValue, '<a href') === FALSE) {
+      $this->format('text/plain')->tokens($entity, $customFieldName, $originalValue);
+    }
 
     return $this->format('text/html')->tokens($entity, $customFieldName, $fieldValue);
   }
@@ -292,7 +298,8 @@ class TokenRow {
         foreach ($htmlTokens as $entity => $values) {
           foreach ($values as $field => $value) {
             if (!$value instanceof \DateTime && !$value instanceof Money) {
-              $value = html_entity_decode(strip_tags($value));
+              // rtrim removes trailing lines from <p> tags.
+              $value = rtrim(\CRM_Utils_String::htmlToText($value));
             }
             if (!isset($textTokens[$entity][$field])) {
               $textTokens[$entity][$field] = $value;
diff --git a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
index fffd067a54..7ab32aac47 100644
--- a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
+++ b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
@@ -1090,7 +1090,14 @@ United States', $tokenProcessor->getRow(0)->render('message'));
 
     // The `description` does allow HTML. Any funny characters are filtered out of text.
     $messages['event_text'] = 'You signed up for this event: {event.title}: {event.description}';
-    $expected['event_text'] = 'You signed up for this event: The Webinar: Some online webinar thingy. Attendees will need to install the TeleFoo app.';
+    $expected['event_text'] = 'You signed up for this event: The Webinar: Some online webinar thingy.
+
+Attendees will need to install the TeleFoo [1] app.
+
+
+Links:
+------
+[1] http://telefoo.example.com';
 
     $messages['event_html'] = '<p>You signed up for this event:</p> <h3>{event.title}</h3> {event.description}';
     $expected['event_html'] = '<p>You signed up for this event:</p> <h3>The Webinar</h3> <p>Some online webinar thingy.</p> <p>Attendees will need to install the <a href="http://telefoo.example.com">TeleFoo</a> app.</p>';
diff --git a/tests/phpunit/Civi/Token/TokenProcessorTest.php b/tests/phpunit/Civi/Token/TokenProcessorTest.php
index f9c747ba6b..c2048cc22c 100644
--- a/tests/phpunit/Civi/Token/TokenProcessorTest.php
+++ b/tests/phpunit/Civi/Token/TokenProcessorTest.php
@@ -1,6 +1,8 @@
 <?php
 namespace Civi\Token;
 
+use Brick\Money\Context\DefaultContext;
+use Brick\Money\Money;
 use Civi\Api4\Website;
 use Civi\Token\Event\TokenRegisterEvent;
 use Civi\Token\Event\TokenValueEvent;
@@ -462,7 +464,7 @@ class TokenProcessorTest extends \CiviUnitTestCase {
     $this->assertEquals(1, $this->counts['onEvalTokens']);
   }
 
-  public function getFilterExamples() {
+  public function getFilterExamples(): array {
     $exampleTokens = [
       // All the "{my_text.*}" tokens will be treated as plain-text ("text/plain").
       'my_text' => [
@@ -477,7 +479,7 @@ class TokenProcessorTest extends \CiviUnitTestCase {
         'and_such' => '<strong>testing &amp; such</strong>',
       ],
       'my_currencies' => [
-        'amount' => \Brick\Money\Money::of(123, 'USD', new \Brick\Money\Context\DefaultContext()),
+        'amount' => Money::of(123, 'USD', new DefaultContext()),
         'currency' => 'EUR',
         'locale' => 'fr_FR',
       ],
@@ -522,7 +524,7 @@ class TokenProcessorTest extends \CiviUnitTestCase {
     $testCases['TextMessages with HtmlData'] = [
       'text/plain',
       [
-        'This is {my_rich_text.and_such}...' => 'This is testing & such...',
+        'This is {my_rich_text.and_such}...' => 'This is TESTING & SUCH...',
         'This is {my_rich_text.and_such|lower}...' => 'This is testing & such...',
         'This is {my_rich_text.and_such|upper}!' => 'This is TESTING & SUCH!',
       ],
-- 
2.25.1