CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * Class to abstract token replacement.
  20  */
  21 class CRM_Utils_Token {
  22   public static $_requiredTokens = NULL;
  23
  24   public static $_tokens = [
  25     'action' => [
  26       'forward',
  27       'optOut',
  28       'optOutUrl',
  29       'reply',
  30       'unsubscribe',
  31       'unsubscribeUrl',
  32       'resubscribe',
  33       'resubscribeUrl',
  34       'subscribeUrl',
  35     ],
  36     'mailing' => [
  37       'id',
  38       'key',
  39       'name',
  40       'group',
  41       'subject',
  42       'viewUrl',
  43       'editUrl',
  44       'scheduleUrl',
  45       'approvalStatus',
  46       'approvalNote',
  47       'approveUrl',
  48       'creator',
  49       'creatorEmail',
  50     ],
  51     'user' => [
  52       // we extract the stuff after the role / permission and return the
  53       // civicrm email addresses of all users with that role / permission
  54       // useful with rules integration
  55       'permission:',
  56       'role:',
  57     ],
  58     // populate this dynamically
  59     'contact' => NULL,
  60     // populate this dynamically
  61     'contribution' => NULL,
  62     'domain' => [
  63       'name',
  64       'phone',
  65       'address',
  66       'email',
  67       'id',
  68       'description',
  69     ],
  70     'subscribe' => ['group'],
  71     'unsubscribe' => ['group'],
  72     'resubscribe' => ['group'],
  73     'welcome' => ['group'],
  74   ];
  75
  76   /**
  77    * @deprecated
  78    *   This is used by CiviMail but will be made redundant by FlexMailer.
  79    * @return array
  80    */
  81   public static function getRequiredTokens() {
  82     if (self::$_requiredTokens == NULL) {
  83       self::$_requiredTokens = [
  84         'domain.address' => ts("Domain address - displays your organization's postal address."),
  85         'action.optOutUrl or action.unsubscribeUrl' => [
  86           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
  87           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
  88           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
  89           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
  90         ],
  91       ];
  92     }
  93     return self::$_requiredTokens;
  94   }
  95
  96   /**
  97    * Check a string (mailing body) for required tokens.
  98    *
  99    * @param string $str
 100    *   The message.
 101    *
 102    * @return bool|array
 103    *   true if all required tokens are found,
 104    *    else an array of the missing tokens
 105    */
 106   public static function requiredTokens(&$str) {
 107     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 108     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, []) : CRM_Utils_Token::getRequiredTokens();
 109
 110     $missing = [];
 111     foreach ($requiredTokens as $token => $value) {
 112       if (!is_array($value)) {
 113         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 114           $missing[$token] = $value;
 115         }
 116       }
 117       else {
 118         $present = FALSE;
 119         $desc = NULL;
 120         foreach ($value as $t => $d) {
 121           $desc = $d;
 122           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 123             $present = TRUE;
 124           }
 125         }
 126         if (!$present) {
 127           $missing[$token] = $desc;
 128         }
 129       }
 130     }
 131
 132     if (empty($missing)) {
 133       return TRUE;
 134     }
 135     return $missing;
 136   }
 137
 138   /**
 139    * Wrapper for token matching.
 140    *
 141    * @param string $type
 142    *   The token type (domain,mailing,contact,action).
 143    * @param string $var
 144    *   The token variable.
 145    * @param string $str
 146    *   The string to search.
 147    *
 148    * @return bool
 149    *   Was there a match
 150    */
 151   public static function token_match($type, $var, &$str) {
 152     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 153     return preg_match("/(^|[^\{])$token/", $str);
 154   }
 155
 156   /**
 157    * Wrapper for token replacing.
 158    *
 159    * @param string $type
 160    *   The token type.
 161    * @param string $var
 162    *   The token variable.
 163    * @param string $value
 164    *   The value to substitute for the token.
 165    * @param string $str (reference) The string to replace in
 166    *
 167    * @param bool $escapeSmarty
 168    *
 169    * @return string
 170    *   The processed string
 171    */
 172   public static function token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 173     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 174     if (!$value) {
 175       $value = '$3';
 176     }
 177     if ($escapeSmarty) {
 178       $value = self::tokenEscapeSmarty($value);
 179     }
 180     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 181     return $str;
 182   }
 183
 184   /**
 185    * Get< the regex for token replacement
 186    *
 187    * @param string $token_type
 188    *   A string indicating the the type of token to be used in the expression.
 189    *
 190    * @return string
 191    *   regular expression sutiable for using in preg_replace
 192    */
 193   private static function tokenRegex($token_type) {
 194     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(\-[\w\s]+)?)\}(?!\})/';
 195   }
 196
 197   /**
 198    * Escape the string so a malicious user cannot inject smarty code into the template.
 199    *
 200    * @param string $string
 201    *   A string that needs to be escaped from smarty parsing.
 202    *
 203    * @return string
 204    *   the escaped string
 205    */
 206   public static function tokenEscapeSmarty($string) {
 207     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 208     return preg_replace(['/{/', '/(?<!{ldelim)}/'], ['{ldelim}', '{rdelim}'], $string);
 209   }
 210
 211   /**
 212    * Replace all the domain-level tokens in $str
 213    *
 214    * @param string $str
 215    *   The string with tokens to be replaced.
 216    * @param object $domain
 217    *   The domain BAO.
 218    * @param bool $html
 219    *   Replace tokens with HTML or plain text.
 220    *
 221    * @param null $knownTokens
 222    * @param bool $escapeSmarty
 223    *
 224    * @return string
 225    *   The processed string
 226    */
 227   public static function replaceDomainTokens(
 228     $str,
 229     $domain,
 230     $html = FALSE,
 231     $knownTokens = NULL,
 232     $escapeSmarty = FALSE
 233   ) {
 234     $key = 'domain';
 235     if (
 236       !$knownTokens || empty($knownTokens[$key])
 237     ) {
 238       return $str;
 239     }
 240
 241     $str = preg_replace_callback(
 242       self::tokenRegex($key),
 243       function ($matches) use ($domain, $html, $escapeSmarty) {
 244         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 245       },
 246       $str
 247     );
 248     return $str;
 249   }
 250
 251   /**
 252    * @param string $token
 253    * @param CRM_Core_BAO_Domain $domain
 254    * @param bool $html
 255    * @param bool $escapeSmarty
 256    *
 257    * @return null|string
 258    */
 259   public static function getDomainTokenReplacement($token, $domain, $html = FALSE, $escapeSmarty = FALSE): ?string {
 260     // check if the token we were passed is valid
 261     // we have to do this because this function is
 262     // called only when we find a token in the string
 263
 264     $loc = $domain->getLocationValues();
 265
 266     if (!in_array($token, self::$_tokens['domain'])) {
 267       $value = "{domain.$token}";
 268     }
 269     elseif ($token === 'address') {
 270       $cacheKey = __CLASS__ . 'address_token_cache' . CRM_Core_Config::domainID();
 271       $addressCache = Civi::cache()->has($cacheKey) ? Civi::cache()->get($cacheKey) : [];
 272
 273       $fieldKey = $html ? 'address-html' : 'address-text';
 274       if (array_key_exists($fieldKey, $addressCache)) {
 275         return $addressCache[$fieldKey];
 276       }
 277
 278       $value = NULL;
 279       // Construct the address token
 280
 281       if (!empty($loc[$token])) {
 282         if ($html) {
 283           $value = str_replace("\n", '<br />', $loc[$token][1]['display']);
 284         }
 285         else {
 286           $value = $loc[$token][1]['display_text'];
 287         }
 288         Civi::cache()->set($cacheKey, $addressCache);
 289       }
 290     }
 291     elseif ($token === 'name' || $token === 'id' || $token === 'description') {
 292       $value = $domain->$token;
 293     }
 294     elseif ($token === 'phone' || $token === 'email') {
 295       // Construct the phone and email tokens
 296
 297       $value = NULL;
 298       if (!empty($loc[$token])) {
 299         foreach ($loc[$token] as $index => $entity) {
 300           $value = $entity[$token];
 301           break;
 302         }
 303       }
 304     }
 305
 306     if ($escapeSmarty) {
 307       $value = self::tokenEscapeSmarty($value);
 308     }
 309
 310     return $value;
 311   }
 312
 313   /**
 314    * Replace all the org-level tokens in $str
 315    *
 316    * @fixme: This function appears to be broken, as it depends on
 317    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 318    * Marking as deprecated until this is fixed
 319    * @deprecated
 320    *
 321    * @param string $str
 322    *   The string with tokens to be replaced.
 323    * @param object $org
 324    *   Associative array of org properties.
 325    * @param bool $html
 326    *   Replace tokens with HTML or plain text.
 327    *
 328    * @param bool $escapeSmarty
 329    *
 330    * @return string
 331    *   The processed string
 332    */
 333   public static function &replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 334     self::$_tokens['org']
 335       = array_merge(
 336         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 337         ['address', 'display_name', 'checksum', 'contact_id']
 338       );
 339
 340     $cv = NULL;
 341     foreach (self::$_tokens['org'] as $token) {
 342       // print "Getting token value for $token<br/><br/>";
 343       if ($token == '') {
 344         continue;
 345       }
 346
 347       // If the string doesn't contain this token, skip it.
 348
 349       if (!self::token_match('org', $token, $str)) {
 350         continue;
 351       }
 352
 353       // Construct value from $token and $contact
 354
 355       $value = NULL;
 356
 357       if ($cfID = CRM_Core_BAO_CustomField::getKeyID($token)) {
 358         // only generate cv if we need it
 359         if ($cv === NULL) {
 360           $cv = CRM_Core_BAO_CustomValue::getContactValues($org['contact_id']);
 361         }
 362         foreach ($cv as $cvFieldID => $value) {
 363           if ($cvFieldID == $cfID) {
 364             $value = CRM_Core_BAO_CustomField::displayValue($value, $cfID);
 365             break;
 366           }
 367         }
 368       }
 369       elseif ($token == 'checksum') {
 370         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 371         $value = "cs={$cs}";
 372       }
 373       elseif ($token == 'address') {
 374         // Build the location values array
 375
 376         $loc = [];
 377         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 378         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 379         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 380         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 381         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 382
 383         // Construct the address token
 384
 385         $value = CRM_Utils_Address::format($loc);
 386         if ($html) {
 387           $value = str_replace("\n", '<br />', $value);
 388         }
 389       }
 390       else {
 391         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 392       }
 393
 394       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 395     }
 396
 397     return $str;
 398   }
 399
 400   /**
 401    * Replace all mailing tokens in $str
 402    *
 403    * @param string $str
 404    *   The string with tokens to be replaced.
 405    * @param object $mailing
 406    *   The mailing BAO, or null for validation.
 407    * @param bool $html
 408    *   Replace tokens with HTML or plain text.
 409    *
 410    * @param null $knownTokens
 411    * @param bool $escapeSmarty
 412    *
 413    * @return string
 414    *   The processed string
 415    */
 416   public static function &replaceMailingTokens(
 417     $str,
 418     &$mailing,
 419     $html = FALSE,
 420     $knownTokens = NULL,
 421     $escapeSmarty = FALSE
 422   ) {
 423     $key = 'mailing';
 424     if (!$knownTokens || !isset($knownTokens[$key])) {
 425       return $str;
 426     }
 427
 428     $str = preg_replace_callback(
 429       self::tokenRegex($key),
 430       function ($matches) use (&$mailing, $escapeSmarty) {
 431         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 432       },
 433       $str
 434     );
 435     return $str;
 436   }
 437
 438   /**
 439    * @param $token
 440    * @param $mailing
 441    * @param bool $escapeSmarty
 442    *
 443    * @return string
 444    */
 445   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 446     $value = '';
 447     switch ($token) {
 448       // CRM-7663
 449
 450       case 'id':
 451         $value = $mailing ? $mailing->id : 'undefined';
 452         break;
 453
 454       // Key is the ID, or the hash when the hash URLs setting is enabled
 455       case 'key':
 456         $value = $mailing->id;
 457         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($value)) {
 458           $value = $hash;
 459         }
 460         break;
 461
 462       case 'name':
 463         $value = $mailing ? $mailing->name : 'Mailing Name';
 464         break;
 465
 466       case 'group':
 467         $groups = $mailing ? $mailing->getGroupNames() : ['Mailing Groups'];
 468         $value = implode(', ', $groups);
 469         break;
 470
 471       case 'subject':
 472         $value = $mailing->subject;
 473         break;
 474
 475       case 'viewUrl':
 476         $mailingKey = $mailing->id;
 477         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 478           $mailingKey = $hash;
 479         }
 480         $value = CRM_Utils_System::url('civicrm/mailing/view',
 481           "reset=1&id={$mailingKey}",
 482           TRUE, NULL, FALSE, TRUE
 483         );
 484         break;
 485
 486       case 'editUrl':
 487       case 'scheduleUrl':
 488         // Note: editUrl and scheduleUrl used to be different, but now there's
 489         // one screen which can adapt based on permissions (in workflow mode).
 490         $value = CRM_Utils_System::url('civicrm/mailing/send',
 491           "reset=1&mid={$mailing->id}&continue=true",
 492           TRUE, NULL, FALSE, TRUE
 493         );
 494         break;
 495
 496       case 'html':
 497         $page = new CRM_Mailing_Page_View();
 498         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 499         break;
 500
 501       case 'approvalStatus':
 502         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 503         break;
 504
 505       case 'approvalNote':
 506         $value = $mailing->approval_note;
 507         break;
 508
 509       case 'approveUrl':
 510         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 511           "reset=1&mid={$mailing->id}",
 512           TRUE, NULL, FALSE, TRUE
 513         );
 514         break;
 515
 516       case 'creator':
 517         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 518         break;
 519
 520       case 'creatorEmail':
 521         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 522         break;
 523
 524       default:
 525         $value = "{mailing.$token}";
 526         break;
 527     }
 528
 529     if ($escapeSmarty) {
 530       $value = self::tokenEscapeSmarty($value);
 531     }
 532     return $value;
 533   }
 534
 535   /**
 536    * Replace all action tokens in $str
 537    *
 538    * @param string $str
 539    *   The string with tokens to be replaced.
 540    * @param array $addresses
 541    *   Assoc. array of VERP event addresses.
 542    * @param array $urls
 543    *   Assoc. array of action URLs.
 544    * @param bool $html
 545    *   Replace tokens with HTML or plain text.
 546    * @param array $knownTokens
 547    *   A list of tokens that are known to exist in the email body.
 548    *
 549    * @param bool $escapeSmarty
 550    *
 551    * @return string
 552    *   The processed string
 553    */
 554   public static function &replaceActionTokens(
 555     $str,
 556     &$addresses,
 557     &$urls,
 558     $html = FALSE,
 559     $knownTokens = NULL,
 560     $escapeSmarty = FALSE
 561   ) {
 562     $key = 'action';
 563     // here we intersect with the list of pre-configured valid tokens
 564     // so that we remove anything we do not recognize
 565     // I hope to move this step out of here soon and
 566     // then we will just iterate on a list of tokens that are passed to us
 567     if (!$knownTokens || empty($knownTokens[$key])) {
 568       return $str;
 569     }
 570
 571     $str = preg_replace_callback(
 572       self::tokenRegex($key),
 573       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 574         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 575       },
 576       $str
 577     );
 578     return $str;
 579   }
 580
 581   /**
 582    * @param $token
 583    * @param $addresses
 584    * @param $urls
 585    * @param bool $html
 586    * @param bool $escapeSmarty
 587    *
 588    * @return mixed|string
 589    */
 590   public static function getActionTokenReplacement(
 591     $token,
 592     &$addresses,
 593     &$urls,
 594     $html = FALSE,
 595     $escapeSmarty = FALSE
 596   ) {
 597     // If the token is an email action, use it.  Otherwise, find the
 598     // appropriate URL.
 599
 600     if (!in_array($token, self::$_tokens['action'])) {
 601       $value = "{action.$token}";
 602     }
 603     else {
 604       $value = $addresses[$token] ?? NULL;
 605
 606       if ($value == NULL) {
 607         $value = $urls[$token] ?? NULL;
 608       }
 609
 610       if ($value && $html) {
 611         // fix for CRM-2318
 612         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 613           $value = "mailto:$value";
 614         }
 615       }
 616       elseif ($value && !$html) {
 617         $value = str_replace('&amp;', '&', $value);
 618       }
 619     }
 620
 621     if ($escapeSmarty) {
 622       $value = self::tokenEscapeSmarty($value);
 623     }
 624     return $value;
 625   }
 626
 627   /**
 628    * Replace all the contact-level tokens in $str with information from
 629    * $contact.
 630    *
 631    * @param string $str
 632    *   The string with tokens to be replaced.
 633    * @param array $contact
 634    *   Associative array of contact properties.
 635    * @param bool $html
 636    *   Replace tokens with HTML or plain text.
 637    * @param array $knownTokens
 638    *   A list of tokens that are known to exist in the email body.
 639    * @param bool $returnBlankToken
 640    *   Return unevaluated token if value is null.
 641    *
 642    * @param bool $escapeSmarty
 643    *
 644    * @return string
 645    *   The processed string
 646    */
 647   public static function replaceContactTokens(
 648     $str,
 649     &$contact,
 650     $html = FALSE,
 651     $knownTokens = NULL,
 652     $returnBlankToken = FALSE,
 653     $escapeSmarty = FALSE
 654   ) {
 655     // Refresh contact tokens in case they have changed. There is heavy caching
 656     // in exportable fields so there is no benefit in doing this conditionally.
 657     self::$_tokens['contact'] = array_merge(
 658       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 659       ['checksum', 'contact_id']
 660     );
 661
 662     $key = 'contact';
 663     // here we intersect with the list of pre-configured valid tokens
 664     // so that we remove anything we do not recognize
 665     // I hope to move this step out of here soon and
 666     // then we will just iterate on a list of tokens that are passed to us
 667     if (!$knownTokens || empty($knownTokens[$key])) {
 668       return $str;
 669     }
 670
 671     $str = preg_replace_callback(
 672       self::tokenRegex($key),
 673       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 674         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 675       },
 676       $str
 677     );
 678
 679     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 680     return $str;
 681   }
 682
 683   /**
 684    * @param $token
 685    * @param $contact
 686    * @param bool $html
 687    * @param bool $returnBlankToken
 688    * @param bool $escapeSmarty
 689    *
 690    * @return bool|mixed|null|string
 691    */
 692   public static function getContactTokenReplacement(
 693     $token,
 694     &$contact,
 695     $html = FALSE,
 696     $returnBlankToken = FALSE,
 697     $escapeSmarty = FALSE
 698   ) {
 699     if (self::$_tokens['contact'] == NULL) {
 700       /* This should come from UF */
 701
 702       self::$_tokens['contact']
 703         = array_merge(
 704           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 705           ['checksum', 'contact_id']
 706         );
 707     }
 708
 709     // Construct value from $token and $contact
 710
 711     $value = NULL;
 712     $noReplace = FALSE;
 713
 714     // Support legacy tokens
 715     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 716
 717     // check if the token we were passed is valid
 718     // we have to do this because this function is
 719     // called only when we find a token in the string
 720
 721     if (!in_array($token, self::$_tokens['contact'])) {
 722       $noReplace = TRUE;
 723     }
 724     elseif ($token == 'checksum') {
 725       $hash = $contact['hash'] ?? NULL;
 726       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 727       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 728         NULL,
 729         NULL,
 730         $hash
 731       );
 732       $value = "cs={$cs}";
 733     }
 734     else {
 735       $value = (array) CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 736
 737       foreach ($value as $index => $item) {
 738         $value[$index] = self::convertPseudoConstantsUsingMetadata($value[$index], $token);
 739       }
 740       $value = implode(', ', $value);
 741     }
 742
 743     if (!$html) {
 744       $value = str_replace('&amp;', '&', $value);
 745     }
 746
 747     // if null then return actual token
 748     if ($returnBlankToken && !$value) {
 749       $noReplace = TRUE;
 750     }
 751
 752     if ($noReplace) {
 753       $value = "{contact.$token}";
 754     }
 755
 756     if ($escapeSmarty
 757       && !($returnBlankToken && $noReplace)
 758     ) {
 759       // $returnBlankToken means the caller wants to do further attempts at
 760       // processing unreplaced tokens -- so don't escape them yet in this case.
 761       $value = self::tokenEscapeSmarty($value);
 762     }
 763
 764     return $value;
 765   }
 766
 767   /**
 768    * Replace all the hook tokens in $str with information from
 769    * $contact.
 770    *
 771    * @param string $str
 772    *   The string with tokens to be replaced.
 773    * @param array $contact
 774    *   Associative array of contact properties (including hook token values).
 775    * @param $categories
 776    * @param bool $html
 777    *   Replace tokens with HTML or plain text.
 778    *
 779    * @param bool $escapeSmarty
 780    *
 781    * @return string
 782    *   The processed string
 783    */
 784   public static function &replaceHookTokens(
 785     $str,
 786     &$contact,
 787     &$categories,
 788     $html = FALSE,
 789     $escapeSmarty = FALSE
 790   ) {
 791     foreach ($categories as $key) {
 792       $str = preg_replace_callback(
 793         self::tokenRegex($key),
 794         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 795           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 796         },
 797         $str
 798       );
 799     }
 800     return $str;
 801   }
 802
 803   /**
 804    * Parse html through Smarty resolving any smarty functions.
 805    * @param string $tokenHtml
 806    * @param array $entity
 807    * @param string $entityType
 808    * @return string
 809    *   html parsed through smarty
 810    */
 811   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 812     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 813       $smarty = CRM_Core_Smarty::singleton();
 814       // also add the tokens to the template
 815       $smarty->assign_by_ref($entityType, $entity);
 816       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 817     }
 818     return $tokenHtml;
 819   }
 820
 821   /**
 822    * @param $token
 823    * @param $contact
 824    * @param $category
 825    * @param bool $html
 826    * @param bool $escapeSmarty
 827    *
 828    * @return mixed|string
 829    */
 830   public static function getHookTokenReplacement(
 831     $token,
 832     &$contact,
 833     $category,
 834     $html = FALSE,
 835     $escapeSmarty = FALSE
 836   ) {
 837     $value = $contact["{$category}.{$token}"] ?? NULL;
 838
 839     if ($value && !$html) {
 840       $value = str_replace('&amp;', '&', $value);
 841     }
 842
 843     if ($escapeSmarty) {
 844       $value = self::tokenEscapeSmarty($value);
 845     }
 846
 847     return $value;
 848   }
 849
 850   /**
 851    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 852    * for example {{token}}  or \{token}  will result in {token} in the final email
 853    *
 854    * this routine will remove the extra backslashes and braces
 855    *
 856    * @param $str ref to the string that will be scanned and modified
 857    */
 858   public static function unescapeTokens(&$str) {
 859     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 860   }
 861
 862   /**
 863    * Replace unsubscribe tokens.
 864    *
 865    * @param string $str
 866    *   The string with tokens to be replaced.
 867    * @param object $domain
 868    *   The domain BAO.
 869    * @param array $groups
 870    *   The groups (if any) being unsubscribed.
 871    * @param bool $html
 872    *   Replace tokens with html or plain text.
 873    * @param int $contact_id
 874    *   The contact ID.
 875    * @param string $hash The security hash of the unsub event
 876    *
 877    * @return string
 878    *   The processed string
 879    */
 880   public static function &replaceUnsubscribeTokens(
 881     $str,
 882     &$domain,
 883     &$groups,
 884     $html,
 885     $contact_id,
 886     $hash
 887   ) {
 888     if (self::token_match('unsubscribe', 'group', $str)) {
 889       if (!empty($groups)) {
 890         $config = CRM_Core_Config::singleton();
 891         $base = CRM_Utils_System::baseURL();
 892
 893         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 894         $dao = new CRM_Contact_DAO_Group();
 895         $dao->find();
 896         while ($dao->fetch()) {
 897           if (substr($dao->visibility, 0, 6) == 'Public') {
 898             $visibleGroups[] = $dao->id;
 899           }
 900         }
 901         $value = implode(', ', $groups);
 902         self::token_replace('unsubscribe', 'group', $value, $str);
 903       }
 904     }
 905     return $str;
 906   }
 907
 908   /**
 909    * Replace resubscribe tokens.
 910    *
 911    * @param string $str
 912    *   The string with tokens to be replaced.
 913    * @param object $domain
 914    *   The domain BAO.
 915    * @param array $groups
 916    *   The groups (if any) being resubscribed.
 917    * @param bool $html
 918    *   Replace tokens with html or plain text.
 919    * @param int $contact_id
 920    *   The contact ID.
 921    * @param string $hash The security hash of the resub event
 922    *
 923    * @return string
 924    *   The processed string
 925    */
 926   public static function &replaceResubscribeTokens(
 927     $str, &$domain, &$groups, $html,
 928     $contact_id, $hash
 929   ) {
 930     if (self::token_match('resubscribe', 'group', $str)) {
 931       if (!empty($groups)) {
 932         $value = implode(', ', $groups);
 933         self::token_replace('resubscribe', 'group', $value, $str);
 934       }
 935     }
 936     return $str;
 937   }
 938
 939   /**
 940    * Replace subscription-confirmation-request tokens
 941    *
 942    * @param string $str
 943    *   The string with tokens to be replaced.
 944    * @param string $group
 945    *   The name of the group being subscribed.
 946    * @param $url
 947    * @param bool $html
 948    *   Replace tokens with html or plain text.
 949    *
 950    * @return string
 951    *   The processed string
 952    */
 953   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 954     if (self::token_match('subscribe', 'group', $str)) {
 955       self::token_replace('subscribe', 'group', $group, $str);
 956     }
 957     if (self::token_match('subscribe', 'url', $str)) {
 958       self::token_replace('subscribe', 'url', $url, $str);
 959     }
 960     return $str;
 961   }
 962
 963   /**
 964    * Replace subscription-invitation tokens
 965    *
 966    * @param string $str
 967    *   The string with tokens to be replaced.
 968    *
 969    * @return string
 970    *   The processed string
 971    */
 972   public static function &replaceSubscribeInviteTokens($str) {
 973     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 974       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 975         'reset=1',
 976         TRUE, NULL, FALSE, TRUE
 977       );
 978       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 979     }
 980
 981     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 982       foreach ($matches as $key => $value) {
 983         $gid = substr($value, 21, -1);
 984         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 985           "reset=1&gid={$gid}",
 986           TRUE, NULL, FALSE, TRUE
 987         );
 988         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
 989       }
 990     }
 991
 992     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
 993       foreach ($matches as $key => $value) {
 994         $gid = substr($value, 18, -1);
 995         $config = CRM_Core_Config::singleton();
 996         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
 997         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
 998         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
 999         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1000       }
1001     }
1002     return $str;
1003   }
1004
1005   /**
1006    * Replace welcome/confirmation tokens
1007    *
1008    * @param string $str
1009    *   The string with tokens to be replaced.
1010    * @param string $group
1011    *   The name of the group being subscribed.
1012    * @param bool $html
1013    *   Replace tokens with html or plain text.
1014    *
1015    * @return string
1016    *   The processed string
1017    */
1018   public static function &replaceWelcomeTokens($str, $group, $html) {
1019     if (self::token_match('welcome', 'group', $str)) {
1020       self::token_replace('welcome', 'group', $group, $str);
1021     }
1022     return $str;
1023   }
1024
1025   /**
1026    * Find unprocessed tokens (call this last)
1027    *
1028    * @param string $str
1029    *   The string to search.
1030    *
1031    * @return array
1032    *   Array of tokens that weren't replaced
1033    */
1034   public static function &unmatchedTokens(&$str) {
1035     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1036     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1037     return $match[1];
1038   }
1039
1040   /**
1041    * Find and replace tokens for each component.
1042    *
1043    * @param string $str
1044    *   The string to search.
1045    * @param array $contact
1046    *   Associative array of contact properties.
1047    * @param array $components
1048    *   A list of tokens that are known to exist in the email body.
1049    *
1050    * @param bool $escapeSmarty
1051    * @param bool $returnEmptyToken
1052    *
1053    * @return string
1054    *   The processed string
1055    */
1056   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1057     if (!is_array($components) || empty($contact)) {
1058       return $str;
1059     }
1060
1061     foreach ($components as $name => $tokens) {
1062       if (!is_array($tokens) || empty($tokens)) {
1063         continue;
1064       }
1065
1066       foreach ($tokens as $token) {
1067         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1068           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1069         }
1070         elseif (!$returnEmptyToken) {
1071           //replacing empty token
1072           self::token_replace($name, $token, "", $str, $escapeSmarty);
1073         }
1074       }
1075     }
1076     return $str;
1077   }
1078
1079   /**
1080    * Get array of string tokens.
1081    *
1082    * @param string $string
1083    *   The input string to parse for tokens.
1084    *
1085    * @return array
1086    *   array of tokens mentioned in field
1087    */
1088   public static function getTokens($string) {
1089     $matches = [];
1090     $tokens = [];
1091     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1092       $string,
1093       $matches,
1094       PREG_PATTERN_ORDER
1095     );
1096
1097     if ($matches[1]) {
1098       foreach ($matches[1] as $token) {
1099         [$type, $name] = preg_split('/\./', $token, 2);
1100         if ($name && $type) {
1101           if (!isset($tokens[$type])) {
1102             $tokens[$type] = [];
1103           }
1104           $tokens[$type][] = $name;
1105         }
1106       }
1107     }
1108     return $tokens;
1109   }
1110
1111   /**
1112    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1113    * and getTokens appears to be historical rather than intentional and should be reviewed
1114    * @param $string
1115    * @return array
1116    *   fields to pass in as return properties when populating token
1117    */
1118   public static function getReturnProperties(&$string) {
1119     $returnProperties = [];
1120     $matches = [];
1121     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1122       $string,
1123       $matches,
1124       PREG_PATTERN_ORDER
1125     );
1126     if ($matches[1]) {
1127       foreach ($matches[1] as $token) {
1128         [$type, $name] = preg_split('/\./', $token, 2);
1129         if ($name) {
1130           $returnProperties["{$name}"] = 1;
1131         }
1132       }
1133     }
1134
1135     return $returnProperties;
1136   }
1137
1138   /**
1139    * Gives required details of contacts in an indexed array format so we
1140    * can iterate in a nice loop and do token evaluation
1141    *
1142    * @param array $contactIDs
1143    * @param array $returnProperties
1144    *   Of required properties.
1145    * @param bool $skipOnHold Don't return on_hold contact info also.
1146    *   Don't return on_hold contact info also.
1147    * @param bool $skipDeceased Don't return deceased contact info.
1148    *   Don't return deceased contact info.
1149    * @param array $extraParams
1150    *   Extra params - DEPRECATED
1151    * @param array $tokens
1152    *   The list of tokens we've extracted from the content.
1153    * @param string|null $className
1154    * @param int|null $jobID
1155    *   The mailing list jobID - this is a legacy param.
1156    *
1157    * @return array - e.g [[1 => ['first_name' => 'bob'...], 34 => ['first_name' => 'fred'...]]]
1158    */
1159   public static function getTokenDetails(
1160     $contactIDs,
1161     $returnProperties = NULL,
1162     $skipOnHold = TRUE,
1163     $skipDeceased = TRUE,
1164     $extraParams = NULL,
1165     $tokens = [],
1166     $className = NULL,
1167     $jobID = NULL
1168   ) {
1169
1170     $params = [];
1171     foreach ($contactIDs as $contactID) {
1172       $params[] = [
1173         CRM_Core_Form::CB_PREFIX . $contactID,
1174         '=',
1175         1,
1176         0,
1177         0,
1178       ];
1179     }
1180
1181     // fix for CRM-2613
1182     if ($skipDeceased) {
1183       $params[] = ['is_deceased', '=', 0, 0, 0];
1184     }
1185
1186     //fix for CRM-3798
1187     if ($skipOnHold) {
1188       $params[] = ['on_hold', '=', 0, 0, 0];
1189     }
1190
1191     if ($extraParams) {
1192       CRM_Core_Error::deprecatedWarning('Passing $extraParams to getTokenDetails() is not supported and will be removed in a future version');
1193       $params = array_merge($params, $extraParams);
1194     }
1195
1196     // if return properties are not passed then get all return properties
1197     if (empty($returnProperties)) {
1198       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1199         ['display_name', 'checksum', 'contact_id']
1200       );
1201       foreach ($fields as $val) {
1202         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1203         // with contacts and are expensive to resolve.
1204         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1205         // and upstream calls to this should populate return properties.
1206         $unavailableFields = ['group', 'tag'];
1207         if (!in_array($val, $unavailableFields)) {
1208           $returnProperties[$val] = 1;
1209         }
1210       }
1211     }
1212
1213     $custom = [];
1214     foreach ($returnProperties as $name => $dontCare) {
1215       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1216       if ($cfID) {
1217         $custom[] = $cfID;
1218       }
1219     }
1220
1221     [$contactDetails] = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1222
1223     foreach ($contactIDs as $contactID) {
1224       if (array_key_exists($contactID, $contactDetails)) {
1225         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1226         ) {
1227           $communicationPreferences = [];
1228           foreach ((array) $contactDetails[$contactID]['preferred_communication_method'] as $val) {
1229             if ($val) {
1230               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1231             }
1232           }
1233           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1234         }
1235
1236         foreach ($custom as $cfID) {
1237           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1238             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1239           }
1240         }
1241
1242         // special case for greeting replacement
1243         foreach ([
1244           'email_greeting',
1245           'postal_greeting',
1246           'addressee',
1247         ] as $val) {
1248           if (!empty($contactDetails[$contactID][$val])) {
1249             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1250           }
1251         }
1252       }
1253     }
1254
1255     // $contactDetails = &$details[0] = is an array of [ contactID => contactDetails ]
1256     // also call a hook and get token details
1257     CRM_Utils_Hook::tokenValues($contactDetails,
1258       $contactIDs,
1259       $jobID,
1260       $tokens,
1261       $className
1262     );
1263     return [$contactDetails];
1264   }
1265
1266   /**
1267    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1268    * specific tokens to be rendered
1269    *
1270    * @param array $contactIDs
1271    *   This should always be array(0) or its not anonymous - left to keep signature same.
1272    *   as main fn
1273    * @param string $returnProperties
1274    * @param bool $skipOnHold
1275    * @param bool $skipDeceased
1276    * @param string $extraParams
1277    * @param array $tokens
1278    * @param string $className
1279    *   Sent as context to the hook.
1280    * @param string $jobID
1281    * @return array
1282    *   contactDetails with hooks swapped out
1283    */
1284   public static function getAnonymousTokenDetails($contactIDs = [0],
1285                                            $returnProperties = NULL,
1286                                            $skipOnHold = TRUE,
1287                                            $skipDeceased = TRUE,
1288                                            $extraParams = NULL,
1289                                            $tokens = [],
1290                                            $className = NULL,
1291                                            $jobID = NULL) {
1292     $details = [0 => []];
1293     // also call a hook and get token details
1294     CRM_Utils_Hook::tokenValues($details[0],
1295       $contactIDs,
1296       $jobID,
1297       $tokens,
1298       $className
1299     );
1300     return $details;
1301   }
1302
1303   /**
1304    * Get Membership Token Details.
1305    * @param array $membershipIDs
1306    *   Array of membership IDS.
1307    */
1308   public static function getMembershipTokenDetails($membershipIDs) {
1309     $memberships = civicrm_api3('membership', 'get', [
1310       'options' => ['limit' => 0],
1311       'membership_id' => ['IN' => (array) $membershipIDs],
1312     ]);
1313     return $memberships['values'];
1314   }
1315
1316   /**
1317    * Replace existing greeting tokens in message/subject.
1318    *
1319    * This function operates by reference, modifying the first parameter. Other
1320    * methods for token replacement in this class return the modified string.
1321    * This leads to inconsistency in how these methods must be applied.
1322    *
1323    * @TODO Remove that inconsistency in usage.
1324    *
1325    * ::replaceContactTokens() may need to be called after this method, to
1326    * replace tokens supplied from this method.
1327    *
1328    * @param string $tokenString
1329    * @param array $contactDetails
1330    * @param int $contactId
1331    * @param string $className
1332    * @param bool $escapeSmarty
1333    */
1334   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1335
1336     if (!$contactDetails && !$contactId) {
1337       return;
1338     }
1339
1340     // check if there are any tokens
1341     $greetingTokens = self::getTokens($tokenString);
1342
1343     if (!empty($greetingTokens)) {
1344       // first use the existing contact object for token replacement
1345       if (!empty($contactDetails)) {
1346         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1347       }
1348
1349       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1350       // check if there are any unevaluated tokens
1351       $greetingTokens = self::getTokens($tokenString);
1352
1353       // $greetingTokens not empty, means there are few tokens which are not
1354       // evaluated, like custom data etc
1355       // so retrieve it from database
1356       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1357         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1358         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1359         $contactParams = ['contact_id' => $contactId];
1360
1361         $greetingDetails = self::getTokenDetails($contactParams,
1362           $greetingsReturnProperties,
1363           FALSE, FALSE, NULL,
1364           $greetingTokens,
1365           $className
1366         );
1367
1368         // again replace tokens
1369         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1370           $greetingDetails,
1371           TRUE,
1372           $greetingTokens,
1373           TRUE,
1374           $escapeSmarty
1375         );
1376       }
1377
1378       // check if there are still any unevaluated tokens
1379       $remainingTokens = self::getTokens($tokenString);
1380
1381       // $greetingTokens not empty, there are customized or hook tokens to replace
1382       if (!empty($remainingTokens)) {
1383         // Fill the return properties array
1384         $greetingTokens = $remainingTokens;
1385         reset($greetingTokens);
1386         $greetingsReturnProperties = [];
1387         foreach ($greetingTokens as $value) {
1388           $props = array_flip($value);
1389           $props = array_fill_keys(array_keys($props), 1);
1390           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1391         }
1392         $contactParams = ['contact_id' => $contactId];
1393         $greetingDetails = self::getTokenDetails($contactParams,
1394           $greetingsReturnProperties,
1395           FALSE, FALSE, NULL,
1396           $greetingTokens,
1397           $className
1398         );
1399         // Prepare variables for calling replaceHookTokens
1400         $categories = array_keys($greetingTokens);
1401         [$contact] = $greetingDetails;
1402         // Replace tokens defined in Hooks.
1403         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1404       }
1405     }
1406   }
1407
1408   /**
1409    * At this point, $contactDetails has loaded the contact from the DAO. Any
1410    * (non-custom) missing fields are null.  By removing them, we can avoid
1411    * expensive calls to CRM_Contact_BAO_Query.
1412    *
1413    * @param string $tokenString
1414    * @param array $contactDetails
1415    * @param array $greetingTokens
1416    */
1417   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1418
1419     // Only applies to contact tokens
1420     if (!array_key_exists('contact', $greetingTokens)) {
1421       return;
1422     }
1423
1424     $greetingTokensOriginal = $greetingTokens;
1425     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1426     // Sometimes contactDetails are in a multidemensional array, sometimes a
1427     // single-dimension array.
1428     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1429       $contactDetails = current($contactDetails[0]);
1430     }
1431     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1432
1433     // Handle legacy tokens
1434     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1435       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1436         $nullFields[] = $oldToken;
1437       }
1438     }
1439
1440     // Remove null contact fields from $greetingTokens
1441     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1442
1443     // Also remove them from $tokenString
1444     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1445     // Handle legacy tokens again, sigh
1446     if (!empty($removedTokens)) {
1447       foreach ($removedTokens as $token) {
1448         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1449           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1450         }
1451       }
1452       foreach ($removedTokens as $token) {
1453         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1454       }
1455     }
1456   }
1457
1458   /**
1459    * @param $tokens
1460    *
1461    * @return array
1462    */
1463   public static function flattenTokens(&$tokens) {
1464     $flattenTokens = [];
1465
1466     foreach ([
1467       'html',
1468       'text',
1469       'subject',
1470     ] as $prop) {
1471       if (!isset($tokens[$prop])) {
1472         continue;
1473       }
1474       foreach ($tokens[$prop] as $type => $names) {
1475         if (!isset($flattenTokens[$type])) {
1476           $flattenTokens[$type] = [];
1477         }
1478         foreach ($names as $name) {
1479           $flattenTokens[$type][$name] = 1;
1480         }
1481       }
1482     }
1483
1484     return $flattenTokens;
1485   }
1486
1487   /**
1488    * Replace all user tokens in $str
1489    *
1490    * @param string $str
1491    *   The string with tokens to be replaced.
1492    *
1493    * @param null $knownTokens
1494    * @param bool $escapeSmarty
1495    *
1496    * @return string
1497    *   The processed string
1498    */
1499   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1500     $key = 'user';
1501     if (!$knownTokens ||
1502       !isset($knownTokens[$key])
1503     ) {
1504       return $str;
1505     }
1506
1507     $str = preg_replace_callback(
1508       self::tokenRegex($key),
1509       function ($matches) use ($escapeSmarty) {
1510         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1511       },
1512       $str
1513     );
1514     return $str;
1515   }
1516
1517   /**
1518    * @param $token
1519    * @param bool $escapeSmarty
1520    *
1521    * @return string
1522    */
1523   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1524     $value = '';
1525
1526     [$objectName, $objectValue] = explode('-', $token, 2);
1527
1528     switch ($objectName) {
1529       case 'permission':
1530         $value = CRM_Core_Permission::permissionEmails($objectValue);
1531         break;
1532
1533       case 'role':
1534         $value = CRM_Core_Permission::roleEmails($objectValue);
1535         break;
1536     }
1537
1538     if ($escapeSmarty) {
1539       $value = self::tokenEscapeSmarty($value);
1540     }
1541
1542     return $value;
1543   }
1544
1545   protected static function _buildContributionTokens() {
1546     $key = 'contribution';
1547     if (self::$_tokens[$key] == NULL) {
1548       self::$_tokens[$key] = array_keys(array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1549         ['campaign', 'financial_type'],
1550         self::getCustomFieldTokens('Contribution')
1551       ));
1552     }
1553   }
1554
1555   /**
1556    * Store membership tokens on the static _tokens array.
1557    */
1558   protected static function _buildMembershipTokens() {
1559     $key = 'membership';
1560     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1561       $membershipTokens = [];
1562       $tokens = CRM_Core_SelectValues::membershipTokens();
1563       foreach ($tokens as $token => $dontCare) {
1564         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1565       }
1566       self::$_tokens[$key] = $membershipTokens;
1567     }
1568   }
1569
1570   /**
1571    * Replace tokens for an entity.
1572    * @param string $entity
1573    * @param array $entityArray
1574    *   (e.g. in format from api).
1575    * @param string $str
1576    *   String to replace in.
1577    * @param array $knownTokens
1578    *   Array of tokens present.
1579    * @param bool $escapeSmarty
1580    * @return string
1581    *   string with replacements made
1582    */
1583   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1584     if (!$knownTokens || empty($knownTokens[$entity])) {
1585       return $str;
1586     }
1587
1588     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1589     $fn = is_callable(['CRM_Utils_Token', $fn]) ? $fn : 'getApiTokenReplacement';
1590     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1591     foreach ($knownTokens[$entity] as $token) {
1592       $replacement = self::$fn($entity, $token, $entityArray);
1593       if ($escapeSmarty) {
1594         $replacement = self::tokenEscapeSmarty($replacement);
1595       }
1596       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1597     }
1598     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1599   }
1600
1601   /**
1602    * @param int $caseId
1603    * @param string $str
1604    * @param array $knownTokens
1605    * @param bool $escapeSmarty
1606    * @return string
1607    * @throws \CiviCRM_API3_Exception
1608    */
1609   public static function replaceCaseTokens($caseId, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1610     if (!$knownTokens || empty($knownTokens['case'])) {
1611       return $str;
1612     }
1613     $case = civicrm_api3('case', 'getsingle', ['id' => $caseId]);
1614     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1615   }
1616
1617   /**
1618    * Generic function for formatting token replacement for an api field
1619    *
1620    * @param string $entity
1621    * @param string $token
1622    * @param array $entityArray
1623    * @return string
1624    * @throws \CiviCRM_API3_Exception
1625    */
1626   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1627     if (!isset($entityArray[$token])) {
1628       return '';
1629     }
1630     $field = civicrm_api3($entity, 'getfield', ['action' => 'get', 'name' => $token, 'get_options' => 'get']);
1631     $field = $field['values'];
1632     $fieldType = $field['type'] ?? NULL;
1633     // Boolean fields
1634     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1635       $field['options'] = [ts('No'), ts('Yes')];
1636     }
1637     // Match pseudoconstants
1638     if (!empty($field['options'])) {
1639       $ret = [];
1640       foreach ((array) $entityArray[$token] as $val) {
1641         $ret[] = $field['options'][$val];
1642       }
1643       return implode(', ', $ret);
1644     }
1645     // Format date fields
1646     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1647       return CRM_Utils_Date::customFormat($entityArray[$token]);
1648     }
1649     return implode(', ', (array) $entityArray[$token]);
1650   }
1651
1652   /**
1653    * Replace Contribution tokens in html.
1654    *
1655    * @param string $str
1656    * @param array $contribution
1657    * @param bool|string $html
1658    * @param string $knownTokens
1659    * @param bool|string $escapeSmarty
1660    *
1661    * @return mixed
1662    */
1663   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1664     $key = 'contribution';
1665     if (!$knownTokens || empty($knownTokens[$key])) {
1666       //early return
1667       return $str;
1668     }
1669     self::_buildContributionTokens();
1670
1671     // here we intersect with the list of pre-configured valid tokens
1672     // so that we remove anything we do not recognize
1673     // I hope to move this step out of here soon and
1674     // then we will just iterate on a list of tokens that are passed to us
1675
1676     $str = preg_replace_callback(
1677       self::tokenRegex($key),
1678       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1679         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1680       },
1681       $str
1682     );
1683
1684     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1685     return $str;
1686   }
1687
1688   /**
1689    * We have a situation where we are rendering more than one token in each field because we are combining
1690    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1691    *
1692    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1693    * it applies (other tokens are OK including date fields)
1694    *
1695    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1696    * does - but the fields is already being formatted through a date function
1697    *
1698    * @param string $separator
1699    * @param string $str
1700    * @param array $contributions
1701    * @param array $knownTokens
1702    *
1703    * @return string
1704    */
1705   public static function replaceMultipleContributionTokens(string $separator, string $str, array $contributions, array $knownTokens): string {
1706     foreach ($knownTokens['contribution'] ?? [] as $token) {
1707       $resolvedTokens = [];
1708       foreach ($contributions as $contribution) {
1709         $resolvedTokens[] = self::replaceContributionTokens('{contribution.' . $token . '}', $contribution, FALSE, $knownTokens);
1710       }
1711       $str = self::token_replace('contribution', $token, implode($separator, $resolvedTokens), $str);
1712     }
1713     return $str;
1714   }
1715
1716   /**
1717    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1718    * - this is used by the pdfLetter task from membership search
1719    * @param string $entity
1720    *   should always be "membership"
1721    * @param string $token
1722    *   field name
1723    * @param array $membership
1724    *   An api result array for a single membership.
1725    * @return string token replacement
1726    */
1727   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1728     self::_buildMembershipTokens();
1729     switch ($token) {
1730       case 'type':
1731         $value = $membership['membership_name'];
1732         break;
1733
1734       case 'status':
1735         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1736         $value = $statuses[$membership['status_id']];
1737         break;
1738
1739       case 'fee':
1740         try {
1741           $value = civicrm_api3('membership_type', 'getvalue', [
1742             'id' => $membership['membership_type_id'],
1743             'return' => 'minimum_fee',
1744           ]);
1745           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1746         }
1747         catch (CiviCRM_API3_Exception $e) {
1748           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1749           // api handles NULL (4.4)
1750           $value = 0;
1751         }
1752         break;
1753
1754       default:
1755         if (in_array($token, self::$_tokens[$entity])) {
1756           $value = $membership[$token];
1757           if (CRM_Utils_String::endsWith($token, '_date')) {
1758             $value = CRM_Utils_Date::customFormat($value);
1759           }
1760         }
1761         else {
1762           // ie unchanged
1763           $value = "{$entity}.{$token}";
1764         }
1765         break;
1766     }
1767
1768     return $value;
1769   }
1770
1771   /**
1772    * @param $token
1773    * @param $contribution
1774    * @param bool $html
1775    * @param bool $escapeSmarty
1776    *
1777    * @return mixed|string
1778    * @throws \CRM_Core_Exception
1779    */
1780   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1781     self::_buildContributionTokens();
1782
1783     switch ($token) {
1784       case 'total_amount':
1785       case 'net_amount':
1786       case 'fee_amount':
1787       case 'non_deductible_amount':
1788         // FIXME: Is this ever a multi-dimensional array?  Why use retrieveValueRecursive()?
1789         $amount = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1790         $currency = CRM_Utils_Array::retrieveValueRecursive($contribution, 'currency');
1791         $value = CRM_Utils_Money::format($amount, $currency);
1792         break;
1793
1794       case 'receive_date':
1795       case 'receipt_date':
1796         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1797         $config = CRM_Core_Config::singleton();
1798         $value = CRM_Utils_Date::customFormat($value, $config->dateformatDatetime);
1799         break;
1800
1801       default:
1802         if (!in_array($token, self::$_tokens['contribution'])) {
1803           $value = "{contribution.$token}";
1804         }
1805         else {
1806           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1807         }
1808         break;
1809     }
1810
1811     if ($escapeSmarty) {
1812       $value = self::tokenEscapeSmarty($value);
1813     }
1814     return $value;
1815   }
1816
1817   /**
1818    * @return array
1819    *   [legacy_token => new_token]
1820    */
1821   public static function legacyContactTokens() {
1822     return [
1823       'individual_prefix' => 'prefix_id',
1824       'individual_suffix' => 'suffix_id',
1825       'gender' => 'gender_id',
1826       'communication_style' => 'communication_style_id',
1827     ];
1828   }
1829
1830   /**
1831    * Get all custom field tokens of $entity
1832    *
1833    * @param string $entity
1834    * @param bool $usedForTokenWidget
1835    *
1836    * @return array
1837    *   return custom field tokens in array('custom_N' => 'label') format
1838    */
1839   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1840     $customTokens = [];
1841     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1842     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1843       $customTokens[sprintf($tokenName, $id)] = $info['label'];
1844     }
1845
1846     return $customTokens;
1847   }
1848
1849   /**
1850    * Formats a token list for the select2 widget
1851    *
1852    * @param $tokens
1853    * @return array
1854    */
1855   public static function formatTokensForDisplay($tokens) {
1856     $sorted = $output = [];
1857
1858     // Sort in ascending order by ignoring word case
1859     natcasesort($tokens);
1860
1861     // Attempt to place tokens into optgroups
1862     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1863     foreach ($tokens as $k => $v) {
1864       // Check to see if this token is already in a group e.g. for custom fields
1865       $split = explode(' :: ', $v);
1866       if (!empty($split[1])) {
1867         $sorted[$split[1]][] = ['id' => $k, 'text' => $split[0]];
1868       }
1869       // Group by entity
1870       else {
1871         $split = explode('.', trim($k, '{}'));
1872         if (isset($split[1])) {
1873           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucwords(str_replace('_', ' ', $split[0]));
1874         }
1875         else {
1876           $entity = 'Contact';
1877         }
1878         $sorted[ts($entity)][] = ['id' => $k, 'text' => $v];
1879       }
1880     }
1881
1882     ksort($sorted);
1883     foreach ($sorted as $k => $v) {
1884       $output[] = ['text' => $k, 'children' => $v];
1885     }
1886
1887     return $output;
1888   }
1889
1890   /**
1891    * @param $value
1892    * @param $token
1893    *
1894    * @return bool|int|mixed|string|null
1895    */
1896   protected static function convertPseudoConstantsUsingMetadata($value, $token) {
1897     // Convert pseudoconstants using metadata
1898     if ($value && is_numeric($value)) {
1899       $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
1900       if (!empty($allFields[$token]['pseudoconstant'])) {
1901         $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
1902       }
1903     }
1904     elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
1905       $value = CRM_Utils_Date::customFormat($value);
1906     }
1907     return $value;
1908   }
1909
1910 }