CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * Class to abstract token replacement.
  20  */
  21 class CRM_Utils_Token {
  22   public static $_requiredTokens = NULL;
  23
  24   public static $_tokens = [
  25     'action' => [
  26       'forward',
  27       'optOut',
  28       'optOutUrl',
  29       'reply',
  30       'unsubscribe',
  31       'unsubscribeUrl',
  32       'resubscribe',
  33       'resubscribeUrl',
  34       'subscribeUrl',
  35     ],
  36     'mailing' => [
  37       'id',
  38       'key',
  39       'name',
  40       'group',
  41       'subject',
  42       'viewUrl',
  43       'editUrl',
  44       'scheduleUrl',
  45       'approvalStatus',
  46       'approvalNote',
  47       'approveUrl',
  48       'creator',
  49       'creatorEmail',
  50     ],
  51     'user' => [
  52       // we extract the stuff after the role / permission and return the
  53       // civicrm email addresses of all users with that role / permission
  54       // useful with rules integration
  55       'permission:',
  56       'role:',
  57     ],
  58     // populate this dynamically
  59     'contact' => NULL,
  60     // populate this dynamically
  61     'contribution' => NULL,
  62     'domain' => [
  63       'name',
  64       'phone',
  65       'address',
  66       'email',
  67       'id',
  68       'description',
  69     ],
  70     'subscribe' => ['group'],
  71     'unsubscribe' => ['group'],
  72     'resubscribe' => ['group'],
  73     'welcome' => ['group'],
  74   ];
  75
  76   /**
  77    * @deprecated
  78    *   This is used by CiviMail but will be made redundant by FlexMailer.
  79    * @return array
  80    */
  81   public static function getRequiredTokens() {
  82     if (self::$_requiredTokens == NULL) {
  83       self::$_requiredTokens = [
  84         'domain.address' => ts("Domain address - displays your organization's postal address."),
  85         'action.optOutUrl or action.unsubscribeUrl' => [
  86           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
  87           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
  88           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
  89           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
  90         ],
  91       ];
  92     }
  93     return self::$_requiredTokens;
  94   }
  95
  96   /**
  97    * Check a string (mailing body) for required tokens.
  98    *
  99    * @param string $str
 100    *   The message.
 101    *
 102    * @return bool|array
 103    *   true if all required tokens are found,
 104    *    else an array of the missing tokens
 105    */
 106   public static function requiredTokens(&$str) {
 107     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 108     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, []) : CRM_Utils_Token::getRequiredTokens();
 109
 110     $missing = [];
 111     foreach ($requiredTokens as $token => $value) {
 112       if (!is_array($value)) {
 113         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 114           $missing[$token] = $value;
 115         }
 116       }
 117       else {
 118         $present = FALSE;
 119         $desc = NULL;
 120         foreach ($value as $t => $d) {
 121           $desc = $d;
 122           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 123             $present = TRUE;
 124           }
 125         }
 126         if (!$present) {
 127           $missing[$token] = $desc;
 128         }
 129       }
 130     }
 131
 132     if (empty($missing)) {
 133       return TRUE;
 134     }
 135     return $missing;
 136   }
 137
 138   /**
 139    * Wrapper for token matching.
 140    *
 141    * @param string $type
 142    *   The token type (domain,mailing,contact,action).
 143    * @param string $var
 144    *   The token variable.
 145    * @param string $str
 146    *   The string to search.
 147    *
 148    * @return bool
 149    *   Was there a match
 150    */
 151   public static function token_match($type, $var, &$str) {
 152     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 153     return preg_match("/(^|[^\{])$token/", $str);
 154   }
 155
 156   /**
 157    * Wrapper for token replacing.
 158    *
 159    * @param string $type
 160    *   The token type.
 161    * @param string $var
 162    *   The token variable.
 163    * @param string $value
 164    *   The value to substitute for the token.
 165    * @param string $str (reference) The string to replace in
 166    *
 167    * @param bool $escapeSmarty
 168    *
 169    * @return string
 170    *   The processed string
 171    */
 172   public static function token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 173     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 174     if (!$value) {
 175       $value = '$3';
 176     }
 177     if ($escapeSmarty) {
 178       $value = self::tokenEscapeSmarty($value);
 179     }
 180     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 181     return $str;
 182   }
 183
 184   /**
 185    * Get the regex for token replacement
 186    *
 187    * @param string $token_type
 188    *   A string indicating the the type of token to be used in the expression.
 189    *
 190    * @return string
 191    *   regular expression suitable for using in preg_replace
 192    */
 193   private static function tokenRegex(string $token_type) {
 194     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(:|\.)?\w*(\-[\w\s]+)?)\}(?!\})/';
 195   }
 196
 197   /**
 198    * Escape the string so a malicious user cannot inject smarty code into the template.
 199    *
 200    * @param string $string
 201    *   A string that needs to be escaped from smarty parsing.
 202    *
 203    * @return string
 204    *   the escaped string
 205    */
 206   public static function tokenEscapeSmarty($string) {
 207     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 208     return preg_replace(['/{/', '/(?<!{ldelim)}/'], ['{ldelim}', '{rdelim}'], $string);
 209   }
 210
 211   /**
 212    * Replace all the domain-level tokens in $str
 213    *
 214    * @param string $str
 215    *   The string with tokens to be replaced.
 216    * @param object $domain
 217    *   The domain BAO.
 218    * @param bool $html
 219    *   Replace tokens with HTML or plain text.
 220    *
 221    * @param null $knownTokens
 222    * @param bool $escapeSmarty
 223    *
 224    * @return string
 225    *   The processed string
 226    */
 227   public static function replaceDomainTokens(
 228     $str,
 229     $domain,
 230     $html = FALSE,
 231     $knownTokens = NULL,
 232     $escapeSmarty = FALSE
 233   ) {
 234     $key = 'domain';
 235     if (
 236       !$knownTokens || empty($knownTokens[$key])
 237     ) {
 238       return $str;
 239     }
 240
 241     $str = preg_replace_callback(
 242       self::tokenRegex($key),
 243       function ($matches) use ($domain, $html, $escapeSmarty) {
 244         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 245       },
 246       $str
 247     );
 248     return $str;
 249   }
 250
 251   /**
 252    * @param string $token
 253    * @param CRM_Core_BAO_Domain $domain
 254    * @param bool $html
 255    * @param bool $escapeSmarty
 256    *
 257    * @return null|string
 258    */
 259   public static function getDomainTokenReplacement($token, $domain, $html = FALSE, $escapeSmarty = FALSE): ?string {
 260     $tokens = CRM_Core_DomainTokens::getDomainTokenValues($domain->id, $html);
 261     $value = $tokens[$token] ?? "{domain.$token}";
 262     if ($escapeSmarty) {
 263       $value = self::tokenEscapeSmarty($value);
 264     }
 265     return $value;
 266   }
 267
 268   /**
 269    * Replace all the org-level tokens in $str
 270    *
 271    * @fixme: This function appears to be broken, as it depended on
 272    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 273    * Marking as deprecated until this is clarified.
 274    *
 275    * @deprecated
 276    *  - the above hard-breakage was there from 2015 to 2021 and
 277    * no error was ever reported on it -does that mean
 278    * 1) the code is never hit because the only function that
 279    * calls this function is never called or
 280    * 2) it was called but never required to resolve any tokens
 281    * or more specifically custom field tokens
 282    *
 283    * The handling for custom fields with the removed token has
 284    * now been removed.
 285    *
 286    * @param string $str
 287    *   The string with tokens to be replaced.
 288    * @param object $org
 289    *   Associative array of org properties.
 290    * @param bool $html
 291    *   Replace tokens with HTML or plain text.
 292    *
 293    * @param bool $escapeSmarty
 294    *
 295    * @return string
 296    *   The processed string
 297    */
 298   public static function replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 299     CRM_Core_Error::deprecatedFunctionWarning('token processor');
 300     self::$_tokens['org']
 301       = array_merge(
 302         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 303         ['address', 'display_name', 'checksum', 'contact_id']
 304       );
 305
 306     foreach (self::$_tokens['org'] as $token) {
 307       // print "Getting token value for $token<br/><br/>";
 308       if ($token === '') {
 309         continue;
 310       }
 311
 312       // If the string doesn't contain this token, skip it.
 313
 314       if (!self::token_match('org', $token, $str)) {
 315         continue;
 316       }
 317
 318       // Construct value from $token and $contact
 319
 320       $value = NULL;
 321
 322       if ($token === 'checksum') {
 323         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 324         $value = "cs={$cs}";
 325       }
 326       elseif ($token === 'address') {
 327         // Build the location values array
 328
 329         $loc = [];
 330         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 331         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 332         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 333         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 334         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 335
 336         // Construct the address token
 337
 338         $value = CRM_Utils_Address::format($loc);
 339         if ($html) {
 340           $value = str_replace("\n", '<br />', $value);
 341         }
 342       }
 343       else {
 344         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 345       }
 346
 347       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 348     }
 349
 350     return $str;
 351   }
 352
 353   /**
 354    * Replace all mailing tokens in $str
 355    *
 356    * @param string $str
 357    *   The string with tokens to be replaced.
 358    * @param object $mailing
 359    *   The mailing BAO, or null for validation.
 360    * @param bool $html
 361    *   Replace tokens with HTML or plain text.
 362    *
 363    * @param null $knownTokens
 364    * @param bool $escapeSmarty
 365    *
 366    * @return string
 367    *   The processed string
 368    */
 369   public static function &replaceMailingTokens(
 370     $str,
 371     &$mailing,
 372     $html = FALSE,
 373     $knownTokens = NULL,
 374     $escapeSmarty = FALSE
 375   ) {
 376     $key = 'mailing';
 377     if (!$knownTokens || !isset($knownTokens[$key])) {
 378       return $str;
 379     }
 380
 381     $str = preg_replace_callback(
 382       self::tokenRegex($key),
 383       function ($matches) use (&$mailing, $escapeSmarty) {
 384         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 385       },
 386       $str
 387     );
 388     return $str;
 389   }
 390
 391   /**
 392    * @param $token
 393    * @param $mailing
 394    * @param bool $escapeSmarty
 395    *
 396    * @return string
 397    */
 398   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 399     $value = '';
 400     switch ($token) {
 401       // CRM-7663
 402
 403       case 'id':
 404         $value = $mailing ? $mailing->id : 'undefined';
 405         break;
 406
 407       // Key is the ID, or the hash when the hash URLs setting is enabled
 408       case 'key':
 409         $value = $mailing->id;
 410         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($value)) {
 411           $value = $hash;
 412         }
 413         break;
 414
 415       case 'name':
 416         $value = $mailing ? $mailing->name : 'Mailing Name';
 417         break;
 418
 419       case 'group':
 420         $groups = $mailing ? $mailing->getGroupNames() : ['Mailing Groups'];
 421         $value = implode(', ', $groups);
 422         break;
 423
 424       case 'subject':
 425         $value = $mailing->subject;
 426         break;
 427
 428       case 'viewUrl':
 429         $mailingKey = $mailing->id;
 430         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 431           $mailingKey = $hash;
 432         }
 433         $value = CRM_Utils_System::url('civicrm/mailing/view',
 434           "reset=1&id={$mailingKey}",
 435           TRUE, NULL, FALSE, TRUE
 436         );
 437         break;
 438
 439       case 'editUrl':
 440       case 'scheduleUrl':
 441         // Note: editUrl and scheduleUrl used to be different, but now there's
 442         // one screen which can adapt based on permissions (in workflow mode).
 443         $value = CRM_Utils_System::url('civicrm/mailing/send',
 444           "reset=1&mid={$mailing->id}&continue=true",
 445           TRUE, NULL, FALSE, TRUE
 446         );
 447         break;
 448
 449       case 'html':
 450         $page = new CRM_Mailing_Page_View();
 451         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 452         break;
 453
 454       case 'approvalStatus':
 455         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 456         break;
 457
 458       case 'approvalNote':
 459         $value = $mailing->approval_note;
 460         break;
 461
 462       case 'approveUrl':
 463         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 464           "reset=1&mid={$mailing->id}",
 465           TRUE, NULL, FALSE, TRUE
 466         );
 467         break;
 468
 469       case 'creator':
 470         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 471         break;
 472
 473       case 'creatorEmail':
 474         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 475         break;
 476
 477       default:
 478         $value = "{mailing.$token}";
 479         break;
 480     }
 481
 482     if ($escapeSmarty) {
 483       $value = self::tokenEscapeSmarty($value);
 484     }
 485     return $value;
 486   }
 487
 488   /**
 489    * Replace all action tokens in $str
 490    *
 491    * @param string $str
 492    *   The string with tokens to be replaced.
 493    * @param array $addresses
 494    *   Assoc. array of VERP event addresses.
 495    * @param array $urls
 496    *   Assoc. array of action URLs.
 497    * @param bool $html
 498    *   Replace tokens with HTML or plain text.
 499    * @param array $knownTokens
 500    *   A list of tokens that are known to exist in the email body.
 501    *
 502    * @param bool $escapeSmarty
 503    *
 504    * @return string
 505    *   The processed string
 506    */
 507   public static function &replaceActionTokens(
 508     $str,
 509     &$addresses,
 510     &$urls,
 511     $html = FALSE,
 512     $knownTokens = NULL,
 513     $escapeSmarty = FALSE
 514   ) {
 515     $key = 'action';
 516     // here we intersect with the list of pre-configured valid tokens
 517     // so that we remove anything we do not recognize
 518     // I hope to move this step out of here soon and
 519     // then we will just iterate on a list of tokens that are passed to us
 520     if (!$knownTokens || empty($knownTokens[$key])) {
 521       return $str;
 522     }
 523
 524     $str = preg_replace_callback(
 525       self::tokenRegex($key),
 526       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 527         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 528       },
 529       $str
 530     );
 531     return $str;
 532   }
 533
 534   /**
 535    * @param $token
 536    * @param $addresses
 537    * @param $urls
 538    * @param bool $html
 539    * @param bool $escapeSmarty
 540    *
 541    * @return mixed|string
 542    */
 543   public static function getActionTokenReplacement(
 544     $token,
 545     &$addresses,
 546     &$urls,
 547     $html = FALSE,
 548     $escapeSmarty = FALSE
 549   ) {
 550     // If the token is an email action, use it.  Otherwise, find the
 551     // appropriate URL.
 552
 553     if (!in_array($token, self::$_tokens['action'])) {
 554       $value = "{action.$token}";
 555     }
 556     else {
 557       $value = $addresses[$token] ?? NULL;
 558
 559       if ($value == NULL) {
 560         $value = $urls[$token] ?? NULL;
 561       }
 562
 563       if ($value && $html) {
 564         // fix for CRM-2318
 565         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 566           $value = "mailto:$value";
 567         }
 568       }
 569       elseif ($value && !$html) {
 570         $value = str_replace('&amp;', '&', $value);
 571       }
 572     }
 573
 574     if ($escapeSmarty) {
 575       $value = self::tokenEscapeSmarty($value);
 576     }
 577     return $value;
 578   }
 579
 580   /**
 581    * Replace all the contact-level tokens in $str with information from
 582    * $contact.
 583    *
 584    * @param string $str
 585    *   The string with tokens to be replaced.
 586    * @param array $contact
 587    *   Associative array of contact properties.
 588    * @param bool $html
 589    *   Replace tokens with HTML or plain text.
 590    * @param array $knownTokens
 591    *   A list of tokens that are known to exist in the email body.
 592    * @param bool $returnBlankToken
 593    *   Return unevaluated token if value is null.
 594    *
 595    * @param bool $escapeSmarty
 596    *
 597    * @return string
 598    *   The processed string
 599    */
 600   public static function replaceContactTokens(
 601     $str,
 602     &$contact,
 603     $html = FALSE,
 604     $knownTokens = NULL,
 605     $returnBlankToken = FALSE,
 606     $escapeSmarty = FALSE
 607   ) {
 608     // Refresh contact tokens in case they have changed. There is heavy caching
 609     // in exportable fields so there is no benefit in doing this conditionally.
 610     self::$_tokens['contact'] = array_merge(
 611       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 612       ['checksum', 'contact_id']
 613     );
 614
 615     $key = 'contact';
 616     // here we intersect with the list of pre-configured valid tokens
 617     // so that we remove anything we do not recognize
 618     // I hope to move this step out of here soon and
 619     // then we will just iterate on a list of tokens that are passed to us
 620     if (!$knownTokens || empty($knownTokens[$key])) {
 621       return $str;
 622     }
 623
 624     $str = preg_replace_callback(
 625       self::tokenRegex($key),
 626       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 627         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 628       },
 629       $str
 630     );
 631
 632     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 633     return $str;
 634   }
 635
 636   /**
 637    * @param $token
 638    * @param $contact
 639    * @param bool $html
 640    * @param bool $returnBlankToken
 641    * @param bool $escapeSmarty
 642    *
 643    * @return bool|mixed|null|string
 644    */
 645   public static function getContactTokenReplacement(
 646     $token,
 647     &$contact,
 648     $html = FALSE,
 649     $returnBlankToken = FALSE,
 650     $escapeSmarty = FALSE
 651   ) {
 652     if (self::$_tokens['contact'] == NULL) {
 653       /* This should come from UF */
 654
 655       self::$_tokens['contact']
 656         = array_merge(
 657           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 658           ['checksum', 'contact_id']
 659         );
 660     }
 661
 662     // Construct value from $token and $contact
 663
 664     $value = NULL;
 665     $noReplace = FALSE;
 666
 667     // Support legacy tokens
 668     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 669
 670     // check if the token we were passed is valid
 671     // we have to do this because this function is
 672     // called only when we find a token in the string
 673
 674     if (!in_array(str_replace(':label', '', $token), self::$_tokens['contact'])) {
 675       $noReplace = TRUE;
 676     }
 677     elseif ($token == 'checksum') {
 678       $hash = $contact['hash'] ?? NULL;
 679       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 680       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 681         NULL,
 682         NULL,
 683         $hash
 684       );
 685       $value = "cs={$cs}";
 686     }
 687     else {
 688       $value = (array) CRM_Utils_Array::retrieveValueRecursive($contact, str_replace(':label', '', $token));
 689
 690       foreach ($value as $index => $item) {
 691         $value[$index] = self::convertPseudoConstantsUsingMetadata($value[$index], str_replace(':label', '', $token));
 692       }
 693       $value = implode(', ', $value);
 694     }
 695
 696     if (!$html) {
 697       $value = str_replace('&amp;', '&', $value);
 698     }
 699
 700     // if null then return actual token
 701     if ($returnBlankToken && !$value) {
 702       $noReplace = TRUE;
 703     }
 704
 705     if ($noReplace) {
 706       $value = "{contact.$token}";
 707     }
 708
 709     if ($escapeSmarty
 710       && !($returnBlankToken && $noReplace)
 711     ) {
 712       // $returnBlankToken means the caller wants to do further attempts at
 713       // processing unreplaced tokens -- so don't escape them yet in this case.
 714       $value = self::tokenEscapeSmarty($value);
 715     }
 716
 717     return $value;
 718   }
 719
 720   /**
 721    * Replace all the hook tokens in $str with information from
 722    * $contact.
 723    *
 724    * @param string $str
 725    *   The string with tokens to be replaced.
 726    * @param array $contact
 727    *   Associative array of contact properties (including hook token values).
 728    * @param $categories
 729    * @param bool $html
 730    *   Replace tokens with HTML or plain text.
 731    *
 732    * @param bool $escapeSmarty
 733    *
 734    * @return string
 735    *   The processed string
 736    */
 737   public static function &replaceHookTokens(
 738     $str,
 739     &$contact,
 740     $categories = NULL,
 741     $html = FALSE,
 742     $escapeSmarty = FALSE
 743   ) {
 744     if (!$categories) {
 745       $categories = self::getTokenCategories();
 746     }
 747     foreach ($categories as $key) {
 748       $str = preg_replace_callback(
 749         self::tokenRegex($key),
 750         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 751           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 752         },
 753         $str
 754       );
 755     }
 756     return $str;
 757   }
 758
 759   /**
 760    * Get the categories required for rendering tokens.
 761    *
 762    * @return array
 763    */
 764   public static function getTokenCategories(): array {
 765     if (!isset(\Civi::$statics[__CLASS__]['token_categories'])) {
 766       $tokens = [];
 767       \CRM_Utils_Hook::tokens($tokens);
 768       \Civi::$statics[__CLASS__]['token_categories'] = array_keys($tokens);
 769     }
 770     return \Civi::$statics[__CLASS__]['token_categories'];
 771   }
 772
 773   /**
 774    * Parse html through Smarty resolving any smarty functions.
 775    * @param string $tokenHtml
 776    * @param array $entity
 777    * @param string $entityType
 778    * @return string
 779    *   html parsed through smarty
 780    * @deprecated
 781    */
 782   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 783     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 784       $smarty = CRM_Core_Smarty::singleton();
 785       // also add the tokens to the template
 786       $smarty->assign_by_ref($entityType, $entity);
 787       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 788     }
 789     return $tokenHtml;
 790   }
 791
 792   /**
 793    * @param $token
 794    * @param $contact
 795    * @param $category
 796    * @param bool $html
 797    * @param bool $escapeSmarty
 798    *
 799    * @return mixed|string
 800    */
 801   public static function getHookTokenReplacement(
 802     $token,
 803     &$contact,
 804     $category,
 805     $html = FALSE,
 806     $escapeSmarty = FALSE
 807   ) {
 808     $value = $contact["{$category}.{$token}"] ?? NULL;
 809
 810     if ($value && !$html) {
 811       $value = str_replace('&amp;', '&', $value);
 812     }
 813
 814     if ($escapeSmarty) {
 815       $value = self::tokenEscapeSmarty($value);
 816     }
 817
 818     return $value;
 819   }
 820
 821   /**
 822    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 823    * for example {{token}}  or \{token}  will result in {token} in the final email
 824    *
 825    * this routine will remove the extra backslashes and braces
 826    *
 827    * @deprecated
 828    *
 829    * @param $str ref to the string that will be scanned and modified
 830    */
 831   public static function unescapeTokens(&$str) {
 832     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 833   }
 834
 835   /**
 836    * Replace unsubscribe tokens.
 837    *
 838    * @param string $str
 839    *   The string with tokens to be replaced.
 840    * @param object $domain
 841    *   The domain BAO.
 842    * @param array $groups
 843    *   The groups (if any) being unsubscribed.
 844    * @param bool $html
 845    *   Replace tokens with html or plain text.
 846    * @param int $contact_id
 847    *   The contact ID.
 848    * @param string $hash The security hash of the unsub event
 849    *
 850    * @return string
 851    *   The processed string
 852    */
 853   public static function &replaceUnsubscribeTokens(
 854     $str,
 855     &$domain,
 856     &$groups,
 857     $html,
 858     $contact_id,
 859     $hash
 860   ) {
 861     if (self::token_match('unsubscribe', 'group', $str)) {
 862       if (!empty($groups)) {
 863         $config = CRM_Core_Config::singleton();
 864         $base = CRM_Utils_System::baseURL();
 865
 866         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 867         $dao = new CRM_Contact_DAO_Group();
 868         $dao->find();
 869         while ($dao->fetch()) {
 870           if (substr($dao->visibility, 0, 6) == 'Public') {
 871             $visibleGroups[] = $dao->id;
 872           }
 873         }
 874         $value = implode(', ', $groups);
 875         self::token_replace('unsubscribe', 'group', $value, $str);
 876       }
 877     }
 878     return $str;
 879   }
 880
 881   /**
 882    * Replace resubscribe tokens.
 883    *
 884    * @param string $str
 885    *   The string with tokens to be replaced.
 886    * @param object $domain
 887    *   The domain BAO.
 888    * @param array $groups
 889    *   The groups (if any) being resubscribed.
 890    * @param bool $html
 891    *   Replace tokens with html or plain text.
 892    * @param int $contact_id
 893    *   The contact ID.
 894    * @param string $hash The security hash of the resub event
 895    *
 896    * @return string
 897    *   The processed string
 898    */
 899   public static function &replaceResubscribeTokens(
 900     $str, &$domain, &$groups, $html,
 901     $contact_id, $hash
 902   ) {
 903     if (self::token_match('resubscribe', 'group', $str)) {
 904       if (!empty($groups)) {
 905         $value = implode(', ', $groups);
 906         self::token_replace('resubscribe', 'group', $value, $str);
 907       }
 908     }
 909     return $str;
 910   }
 911
 912   /**
 913    * Replace subscription-confirmation-request tokens
 914    *
 915    * @param string $str
 916    *   The string with tokens to be replaced.
 917    * @param string $group
 918    *   The name of the group being subscribed.
 919    * @param $url
 920    * @param bool $html
 921    *   Replace tokens with html or plain text.
 922    *
 923    * @return string
 924    *   The processed string
 925    */
 926   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 927     if (self::token_match('subscribe', 'group', $str)) {
 928       self::token_replace('subscribe', 'group', $group, $str);
 929     }
 930     if (self::token_match('subscribe', 'url', $str)) {
 931       self::token_replace('subscribe', 'url', $url, $str);
 932     }
 933     return $str;
 934   }
 935
 936   /**
 937    * Replace subscription-invitation tokens
 938    *
 939    * @param string $str
 940    *   The string with tokens to be replaced.
 941    *
 942    * @return string
 943    *   The processed string
 944    */
 945   public static function &replaceSubscribeInviteTokens($str) {
 946     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 947       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 948         'reset=1',
 949         TRUE, NULL, FALSE, TRUE
 950       );
 951       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 952     }
 953
 954     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 955       foreach ($matches as $key => $value) {
 956         $gid = substr($value, 21, -1);
 957         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 958           "reset=1&gid={$gid}",
 959           TRUE, NULL, FALSE, TRUE
 960         );
 961         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
 962       }
 963     }
 964
 965     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
 966       foreach ($matches as $key => $value) {
 967         $gid = substr($value, 18, -1);
 968         $config = CRM_Core_Config::singleton();
 969         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
 970         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
 971         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
 972         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
 973       }
 974     }
 975     return $str;
 976   }
 977
 978   /**
 979    * Replace welcome/confirmation tokens
 980    *
 981    * @param string $str
 982    *   The string with tokens to be replaced.
 983    * @param string $group
 984    *   The name of the group being subscribed.
 985    * @param bool $html
 986    *   Replace tokens with html or plain text.
 987    *
 988    * @return string
 989    *   The processed string
 990    */
 991   public static function &replaceWelcomeTokens($str, $group, $html) {
 992     if (self::token_match('welcome', 'group', $str)) {
 993       self::token_replace('welcome', 'group', $group, $str);
 994     }
 995     return $str;
 996   }
 997
 998   /**
 999    * Find unprocessed tokens (call this last)
1000    *
1001    * @param string $str
1002    *   The string to search.
1003    *
1004    * @return array
1005    *   Array of tokens that weren't replaced
1006    */
1007   public static function &unmatchedTokens(&$str) {
1008     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1009     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1010     return $match[1];
1011   }
1012
1013   /**
1014    * Find and replace tokens for each component.
1015    *
1016    * @param string $str
1017    *   The string to search.
1018    * @param array $contact
1019    *   Associative array of contact properties.
1020    * @param array $components
1021    *   A list of tokens that are known to exist in the email body.
1022    *
1023    * @param bool $escapeSmarty
1024    * @param bool $returnEmptyToken
1025    *
1026    * @return string
1027    *   The processed string
1028    *
1029    * @deprecated
1030    */
1031   public static function replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1032     CRM_Core_Error::deprecatedFunctionWarning('use the token processor');
1033     if (!is_array($components) || empty($contact)) {
1034       return $str;
1035     }
1036
1037     foreach ($components as $name => $tokens) {
1038       if (!is_array($tokens) || empty($tokens)) {
1039         continue;
1040       }
1041
1042       foreach ($tokens as $token) {
1043         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1044           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1045         }
1046         elseif (!$returnEmptyToken) {
1047           //replacing empty token
1048           self::token_replace($name, $token, "", $str, $escapeSmarty);
1049         }
1050       }
1051     }
1052     return $str;
1053   }
1054
1055   /**
1056    * Get array of string tokens.
1057    *
1058    * @param string $string
1059    *   The input string to parse for tokens.
1060    *
1061    * @return array
1062    *   array of tokens mentioned in field
1063    */
1064   public static function getTokens($string) {
1065     $matches = [];
1066     $tokens = [];
1067     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+(:|.)?\w*)\}(?!\})/',
1068       $string,
1069       $matches,
1070       PREG_PATTERN_ORDER
1071     );
1072
1073     if ($matches[1]) {
1074       foreach ($matches[1] as $token) {
1075         $parts = explode('.', $token, 3);
1076         $type = $parts[0];
1077         $name = $parts[1];
1078         $suffix = !empty($parts[2]) ? ('.' . $parts[2]) : '';
1079         if ($name && $type) {
1080           if (!isset($tokens[$type])) {
1081             $tokens[$type] = [];
1082           }
1083           $tokens[$type][] = $name . $suffix;
1084         }
1085       }
1086     }
1087     return $tokens;
1088   }
1089
1090   /**
1091    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1092    * and getTokens appears to be historical rather than intentional and should be reviewed
1093    * @param $string
1094    * @return array
1095    *   fields to pass in as return properties when populating token
1096    */
1097   public static function getReturnProperties(&$string) {
1098     $returnProperties = [];
1099     $matches = [];
1100     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1101       $string,
1102       $matches,
1103       PREG_PATTERN_ORDER
1104     );
1105     if ($matches[1]) {
1106       foreach ($matches[1] as $token) {
1107         [$type, $name] = preg_split('/\./', $token, 2);
1108         if ($name) {
1109           $returnProperties["{$name}"] = 1;
1110         }
1111       }
1112     }
1113
1114     return $returnProperties;
1115   }
1116
1117   /**
1118    * Gives required details of contacts in an indexed array format so we
1119    * can iterate in a nice loop and do token evaluation
1120    *
1121    * @param array $contactIDs
1122    * @param array $returnProperties
1123    *   Of required properties.
1124    * @param bool $skipOnHold Don't return on_hold contact info also.
1125    *   Don't return on_hold contact info also.
1126    * @param bool $skipDeceased Don't return deceased contact info.
1127    *   Don't return deceased contact info.
1128    * @param array $extraParams
1129    *   Extra params - DEPRECATED
1130    * @param array $tokens
1131    *   The list of tokens we've extracted from the content.
1132    * @param string|null $className
1133    * @param int|null $jobID
1134    *   The mailing list jobID - this is a legacy param.
1135    *
1136    * @return array - e.g [[1 => ['first_name' => 'bob'...], 34 => ['first_name' => 'fred'...]]]
1137    */
1138   public static function getTokenDetails(
1139     $contactIDs,
1140     $returnProperties = NULL,
1141     $skipOnHold = TRUE,
1142     $skipDeceased = TRUE,
1143     $extraParams = NULL,
1144     $tokens = [],
1145     $className = NULL,
1146     $jobID = NULL
1147   ) {
1148
1149     $params = [];
1150     foreach ($contactIDs as $contactID) {
1151       $params[] = [
1152         CRM_Core_Form::CB_PREFIX . $contactID,
1153         '=',
1154         1,
1155         0,
1156         0,
1157       ];
1158     }
1159
1160     // fix for CRM-2613
1161     if ($skipDeceased) {
1162       $params[] = ['is_deceased', '=', 0, 0, 0];
1163     }
1164
1165     //fix for CRM-3798
1166     if ($skipOnHold) {
1167       $params[] = ['on_hold', '=', 0, 0, 0];
1168     }
1169
1170     if ($extraParams) {
1171       CRM_Core_Error::deprecatedWarning('Passing $extraParams to getTokenDetails() is not supported and will be removed in a future version');
1172       $params = array_merge($params, $extraParams);
1173     }
1174
1175     // if return properties are not passed then get all return properties
1176     if (empty($returnProperties)) {
1177       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1178         ['display_name', 'checksum', 'contact_id']
1179       );
1180       foreach ($fields as $val) {
1181         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1182         // with contacts and are expensive to resolve.
1183         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1184         // and upstream calls to this should populate return properties.
1185         $unavailableFields = ['group', 'tag'];
1186         if (!in_array($val, $unavailableFields)) {
1187           $returnProperties[$val] = 1;
1188         }
1189       }
1190     }
1191
1192     $custom = [];
1193     foreach ($returnProperties as $name => $dontCare) {
1194       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1195       if ($cfID) {
1196         $custom[] = $cfID;
1197       }
1198     }
1199
1200     [$contactDetails] = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1201
1202     foreach ($contactIDs as $contactID) {
1203       if (array_key_exists($contactID, $contactDetails)) {
1204         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1205         ) {
1206           $communicationPreferences = [];
1207           foreach ((array) $contactDetails[$contactID]['preferred_communication_method'] as $val) {
1208             if ($val) {
1209               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1210             }
1211           }
1212           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1213         }
1214
1215         foreach ($custom as $cfID) {
1216           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1217             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1218           }
1219         }
1220
1221         // special case for greeting replacement
1222         foreach ([
1223           'email_greeting',
1224           'postal_greeting',
1225           'addressee',
1226         ] as $val) {
1227           if (!empty($contactDetails[$contactID][$val])) {
1228             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1229           }
1230         }
1231       }
1232     }
1233
1234     // $contactDetails = &$details[0] = is an array of [ contactID => contactDetails ]
1235     // also call a hook and get token details
1236     CRM_Utils_Hook::tokenValues($contactDetails,
1237       $contactIDs,
1238       $jobID,
1239       $tokens,
1240       $className
1241     );
1242     return [$contactDetails];
1243   }
1244
1245   /**
1246    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1247    * specific tokens to be rendered
1248    *
1249    * @param array $contactIDs
1250    *   This should always be array(0) or its not anonymous - left to keep signature same.
1251    *   as main fn
1252    * @param string $returnProperties
1253    * @param bool $skipOnHold
1254    * @param bool $skipDeceased
1255    * @param string $extraParams
1256    * @param array $tokens
1257    * @param string $className
1258    *   Sent as context to the hook.
1259    * @param string $jobID
1260    * @return array
1261    *   contactDetails with hooks swapped out
1262    */
1263   public static function getAnonymousTokenDetails($contactIDs = [0],
1264                                            $returnProperties = NULL,
1265                                            $skipOnHold = TRUE,
1266                                            $skipDeceased = TRUE,
1267                                            $extraParams = NULL,
1268                                            $tokens = [],
1269                                            $className = NULL,
1270                                            $jobID = NULL) {
1271     $details = [0 => []];
1272     // also call a hook and get token details
1273     CRM_Utils_Hook::tokenValues($details[0],
1274       $contactIDs,
1275       $jobID,
1276       $tokens,
1277       $className
1278     );
1279     return $details;
1280   }
1281
1282   /**
1283    * Get Membership Token Details.
1284    * @param array $membershipIDs
1285    *   Array of membership IDS.
1286    *
1287    * @deprecated
1288    */
1289   public static function getMembershipTokenDetails($membershipIDs) {
1290     $memberships = civicrm_api3('membership', 'get', [
1291       'options' => ['limit' => 0],
1292       'membership_id' => ['IN' => (array) $membershipIDs],
1293     ]);
1294     return $memberships['values'];
1295   }
1296
1297   /**
1298    * Replace existing greeting tokens in message/subject.
1299    *
1300    * This function operates by reference, modifying the first parameter. Other
1301    * methods for token replacement in this class return the modified string.
1302    * This leads to inconsistency in how these methods must be applied.
1303    *
1304    * @TODO Remove that inconsistency in usage.
1305    *
1306    * ::replaceContactTokens() may need to be called after this method, to
1307    * replace tokens supplied from this method.
1308    *
1309    * @param string $tokenString
1310    * @param array $contactDetails
1311    * @param int $contactId
1312    * @param string $className
1313    * @param bool $escapeSmarty
1314    */
1315   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1316
1317     if (!$contactDetails && !$contactId) {
1318       return;
1319     }
1320
1321     // check if there are any tokens
1322     $greetingTokens = self::getTokens($tokenString);
1323
1324     if (!empty($greetingTokens)) {
1325       // first use the existing contact object for token replacement
1326       if (!empty($contactDetails)) {
1327         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1328       }
1329
1330       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1331       // check if there are any unevaluated tokens
1332       $greetingTokens = self::getTokens($tokenString);
1333
1334       // $greetingTokens not empty, means there are few tokens which are not
1335       // evaluated, like custom data etc
1336       // so retrieve it from database
1337       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1338         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1339         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1340         $contactParams = ['contact_id' => $contactId];
1341
1342         $greetingDetails = self::getTokenDetails($contactParams,
1343           $greetingsReturnProperties,
1344           FALSE, FALSE, NULL,
1345           $greetingTokens,
1346           $className
1347         );
1348
1349         // again replace tokens
1350         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1351           $greetingDetails,
1352           TRUE,
1353           $greetingTokens,
1354           TRUE,
1355           $escapeSmarty
1356         );
1357       }
1358
1359       // check if there are still any unevaluated tokens
1360       $remainingTokens = self::getTokens($tokenString);
1361
1362       // $greetingTokens not empty, there are customized or hook tokens to replace
1363       if (!empty($remainingTokens)) {
1364         // Fill the return properties array
1365         $greetingTokens = $remainingTokens;
1366         reset($greetingTokens);
1367         $greetingsReturnProperties = [];
1368         foreach ($greetingTokens as $value) {
1369           $props = array_flip($value);
1370           $props = array_fill_keys(array_keys($props), 1);
1371           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1372         }
1373         $contactParams = ['contact_id' => $contactId];
1374         $greetingDetails = self::getTokenDetails($contactParams,
1375           $greetingsReturnProperties,
1376           FALSE, FALSE, NULL,
1377           $greetingTokens,
1378           $className
1379         );
1380         // Prepare variables for calling replaceHookTokens
1381         $categories = array_keys($greetingTokens);
1382         [$contact] = $greetingDetails;
1383         // Replace tokens defined in Hooks.
1384         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1385       }
1386     }
1387   }
1388
1389   /**
1390    * At this point, $contactDetails has loaded the contact from the DAO. Any
1391    * (non-custom) missing fields are null.  By removing them, we can avoid
1392    * expensive calls to CRM_Contact_BAO_Query.
1393    *
1394    * @param string $tokenString
1395    * @param array $contactDetails
1396    * @param array $greetingTokens
1397    */
1398   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1399
1400     // Only applies to contact tokens
1401     if (!array_key_exists('contact', $greetingTokens)) {
1402       return;
1403     }
1404
1405     $greetingTokensOriginal = $greetingTokens;
1406     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1407     // Sometimes contactDetails are in a multidemensional array, sometimes a
1408     // single-dimension array.
1409     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1410       $contactDetails = current($contactDetails[0]);
1411     }
1412     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1413
1414     // Handle legacy tokens
1415     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1416       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1417         $nullFields[] = $oldToken;
1418       }
1419     }
1420
1421     // Remove null contact fields from $greetingTokens
1422     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1423
1424     // Also remove them from $tokenString
1425     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1426     // Handle legacy tokens again, sigh
1427     if (!empty($removedTokens)) {
1428       foreach ($removedTokens as $token) {
1429         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1430           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1431         }
1432       }
1433       foreach ($removedTokens as $token) {
1434         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1435       }
1436     }
1437   }
1438
1439   /**
1440    * @param $tokens
1441    *
1442    * @return array
1443    */
1444   public static function flattenTokens(&$tokens) {
1445     $flattenTokens = [];
1446
1447     foreach ([
1448       'html',
1449       'text',
1450       'subject',
1451     ] as $prop) {
1452       if (!isset($tokens[$prop])) {
1453         continue;
1454       }
1455       foreach ($tokens[$prop] as $type => $names) {
1456         if (!isset($flattenTokens[$type])) {
1457           $flattenTokens[$type] = [];
1458         }
1459         foreach ($names as $name) {
1460           $flattenTokens[$type][$name] = 1;
1461         }
1462       }
1463     }
1464
1465     return $flattenTokens;
1466   }
1467
1468   /**
1469    * Replace all user tokens in $str
1470    *
1471    * @param string $str
1472    *   The string with tokens to be replaced.
1473    *
1474    * @param null $knownTokens
1475    * @param bool $escapeSmarty
1476    *
1477    * @return string
1478    *   The processed string
1479    */
1480   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1481     $key = 'user';
1482     if (!$knownTokens ||
1483       !isset($knownTokens[$key])
1484     ) {
1485       return $str;
1486     }
1487
1488     $str = preg_replace_callback(
1489       self::tokenRegex($key),
1490       function ($matches) use ($escapeSmarty) {
1491         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1492       },
1493       $str
1494     );
1495     return $str;
1496   }
1497
1498   /**
1499    * @param $token
1500    * @param bool $escapeSmarty
1501    *
1502    * @return string
1503    */
1504   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1505     $value = '';
1506
1507     [$objectName, $objectValue] = explode('-', $token, 2);
1508
1509     switch ($objectName) {
1510       case 'permission':
1511         $value = CRM_Core_Permission::permissionEmails($objectValue);
1512         break;
1513
1514       case 'role':
1515         $value = CRM_Core_Permission::roleEmails($objectValue);
1516         break;
1517     }
1518
1519     if ($escapeSmarty) {
1520       $value = self::tokenEscapeSmarty($value);
1521     }
1522
1523     return $value;
1524   }
1525
1526   /**
1527    * @deprecated
1528    *
1529    * Do not use this function - it still needs full removal from active code
1530    * in CRM_Contribute_Form_Task_PDFLetter.
1531    */
1532   protected static function _buildContributionTokens() {
1533     $key = 'contribution';
1534
1535     if (!isset(Civi::$statics[__CLASS__][__FUNCTION__][$key])) {
1536       $tokens = array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1537         ['campaign' => [], 'financial_type' => [], 'payment_instrument' => []],
1538         self::getCustomFieldTokens('Contribution'),
1539         [
1540           'financial_type_id:label',
1541           'financial_type_id:name',
1542           'contribution_page_id:label',
1543           'contribution_page_id:name',
1544           'payment_instrument_id:label',
1545           'payment_instrument_id:name',
1546           'is_test:label',
1547           'is_pay_later:label',
1548           'contribution_status_id:label',
1549           'contribution_status_id:name',
1550           'is_template:label',
1551         ]
1552       );
1553       foreach ($tokens as $token) {
1554         if (!empty($token['name'])) {
1555           $tokens[$token['name']] = [];
1556         }
1557       }
1558       Civi::$statics[__CLASS__][__FUNCTION__][$key] = array_keys($tokens);
1559     }
1560     self::$_tokens[$key] = Civi::$statics[__CLASS__][__FUNCTION__][$key];
1561   }
1562
1563   /**
1564    * Do not use.
1565    *
1566    * @deprecated
1567    *
1568    * Replace tokens for an entity.
1569    * @param string $entity
1570    * @param array $entityArray
1571    *   (e.g. in format from api).
1572    * @param string $str
1573    *   String to replace in.
1574    * @param array $knownTokens
1575    *   Array of tokens present.
1576    * @param bool $escapeSmarty
1577    * @return string
1578    *   string with replacements made
1579    */
1580   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1581     if (!$knownTokens || empty($knownTokens[$entity])) {
1582       return $str;
1583     }
1584
1585     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1586     $fn = is_callable(['CRM_Utils_Token', $fn]) ? $fn : 'getApiTokenReplacement';
1587     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1588     foreach ($knownTokens[$entity] as $token) {
1589       // We are now supporting the syntax case_type_id:label
1590       // so strip anything after the ':'
1591       // (we aren't supporting 'name' at this stage, so we can assume 'label'
1592       // test cover in TokenConsistencyTest.
1593       $parts = explode(':', $token);
1594       $replacement = self::$fn($entity, $parts[0], $entityArray);
1595       if ($escapeSmarty) {
1596         $replacement = self::tokenEscapeSmarty($replacement);
1597       }
1598       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1599     }
1600     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1601   }
1602
1603   /**
1604    * @deprecated
1605    *
1606    * @param int $caseId
1607    * @param string $str
1608    * @param array $knownTokens
1609    * @param bool $escapeSmarty
1610    * @return string
1611    * @throws \CiviCRM_API3_Exception
1612    */
1613   public static function replaceCaseTokens($caseId, $str, $knownTokens = NULL, $escapeSmarty = FALSE): string {
1614     if (strpos($str, '{case.') === FALSE) {
1615       return $str;
1616     }
1617     if (!$knownTokens) {
1618       $knownTokens = self::getTokens($str);
1619     }
1620     $case = civicrm_api3('case', 'getsingle', ['id' => $caseId]);
1621     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1622   }
1623
1624   /**
1625    * Generic function for formatting token replacement for an api field
1626    *
1627    * @deprecated
1628    *
1629    * @param string $entity
1630    * @param string $token
1631    * @param array $entityArray
1632    * @return string
1633    * @throws \CiviCRM_API3_Exception
1634    */
1635   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1636     if (!isset($entityArray[$token])) {
1637       return '';
1638     }
1639     $field = civicrm_api3($entity, 'getfield', ['action' => 'get', 'name' => $token, 'get_options' => 'get']);
1640     $field = $field['values'];
1641     $fieldType = $field['type'] ?? NULL;
1642     // Boolean fields
1643     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1644       $field['options'] = [ts('No'), ts('Yes')];
1645     }
1646     // Match pseudoconstants
1647     if (!empty($field['options'])) {
1648       $ret = [];
1649       foreach ((array) $entityArray[$token] as $val) {
1650         $ret[] = $field['options'][$val];
1651       }
1652       return implode(', ', $ret);
1653     }
1654     // Format date fields
1655     elseif ($entityArray[$token] && in_array($fieldType, [CRM_Utils_Type::T_DATE, CRM_Utils_Type::T_TIMESTAMP, (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME)])) {
1656       return CRM_Utils_Date::customFormat($entityArray[$token]);
1657     }
1658     return implode(', ', (array) $entityArray[$token]);
1659   }
1660
1661   /**
1662    * Do not use - unused in core.
1663    *
1664    * Replace Contribution tokens in html.
1665    *
1666    * @param string $str
1667    * @param array $contribution
1668    * @param bool|string $html
1669    * @param string $knownTokens
1670    * @param bool|string $escapeSmarty
1671    *
1672    * @deprecated
1673    *
1674    * @return mixed
1675    */
1676   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1677     $key = 'contribution';
1678     if (!$knownTokens || empty($knownTokens[$key])) {
1679       //early return
1680       return $str;
1681     }
1682
1683     // here we intersect with the list of pre-configured valid tokens
1684     // so that we remove anything we do not recognize
1685     // I hope to move this step out of here soon and
1686     // then we will just iterate on a list of tokens that are passed to us
1687
1688     $str = preg_replace_callback(
1689       self::tokenRegex($key),
1690       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1691         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1692       },
1693       $str
1694     );
1695
1696     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1697     return $str;
1698   }
1699
1700   /**
1701    * We have a situation where we are rendering more than one token in each field because we are combining
1702    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1703    *
1704    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1705    * it applies (other tokens are OK including date fields)
1706    *
1707    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1708    * does - but the fields is already being formatted through a date function
1709    *
1710    * @param string $separator
1711    * @param string $str
1712    * @param array $contributions
1713    * @param array $knownTokens
1714    *
1715    * @deprecated
1716    *
1717    * @return string
1718    */
1719   public static function replaceMultipleContributionTokens(string $separator, string $str, array $contributions, array $knownTokens): string {
1720     CRM_Core_Error::deprecatedFunctionWarning('no alternative');
1721     foreach ($knownTokens['contribution'] ?? [] as $token) {
1722       $resolvedTokens = [];
1723       foreach ($contributions as $contribution) {
1724         $resolvedTokens[] = self::replaceContributionTokens('{contribution.' . $token . '}', $contribution, FALSE, $knownTokens);
1725       }
1726       $str = self::token_replace('contribution', $token, implode($separator, $resolvedTokens), $str);
1727     }
1728     return $str;
1729   }
1730
1731   /**
1732    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1733    * - this is used by the pdfLetter task from membership search
1734    *
1735    * This is called via replaceEntityTokens.
1736    *
1737    * In the near term it will not be called at all from core as
1738    * the pdf letter task is updated to use the processor.
1739    *
1740    * @deprecated
1741    *
1742    * @param string $entity
1743    *   should always be "membership"
1744    * @param string $token
1745    *   field name
1746    * @param array $membership
1747    *   An api result array for a single membership.
1748    * @return string token replacement
1749    */
1750   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1751     $supportedTokens = [
1752       'id',
1753       'status',
1754       'status_id',
1755       'type',
1756       'membership_type_id',
1757       'start_date',
1758       'join_date',
1759       'end_date',
1760       'fee',
1761     ];
1762     switch ($token) {
1763       case 'type':
1764         // membership_type_id would only be requested if the calling
1765         // class is mapping it to '{membership:membership_type_id:label'}
1766       case 'membership_type_id':
1767         $value = $membership['membership_name'];
1768         break;
1769
1770       case 'status':
1771         // status_id would only be requested if the calling
1772         // class is mapping it to '{membership:status_id:label'}
1773       case 'status_id':
1774         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1775         $value = $statuses[$membership['status_id']];
1776         break;
1777
1778       case 'fee':
1779         try {
1780           $value = civicrm_api3('membership_type', 'getvalue', [
1781             'id' => $membership['membership_type_id'],
1782             'return' => 'minimum_fee',
1783           ]);
1784           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1785         }
1786         catch (CiviCRM_API3_Exception $e) {
1787           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1788           // api handles NULL (4.4)
1789           $value = 0;
1790         }
1791         break;
1792
1793       default:
1794         if (in_array($token, $supportedTokens)) {
1795           $value = $membership[$token];
1796           if (CRM_Utils_String::endsWith($token, '_date')) {
1797             $value = CRM_Utils_Date::customFormat($value);
1798           }
1799         }
1800         else {
1801           // ie unchanged
1802           $value = "{$entity}.{$token}";
1803         }
1804         break;
1805     }
1806
1807     return $value;
1808   }
1809
1810   /**
1811    * Do not use - unused in core.
1812    *
1813    * @param $token
1814    * @param $contribution
1815    * @param bool $html
1816    * @param bool $escapeSmarty
1817    *
1818    * @deprecated
1819    *
1820    * @return mixed|string
1821    * @throws \CRM_Core_Exception
1822    */
1823   public static function getContributionTokenReplacement($token, $contribution, $html = FALSE, $escapeSmarty = FALSE) {
1824     self::_buildContributionTokens();
1825
1826     switch ($token) {
1827       case 'total_amount':
1828       case 'net_amount':
1829       case 'fee_amount':
1830       case 'non_deductible_amount':
1831         // FIXME: Is this ever a multi-dimensional array?  Why use retrieveValueRecursive()?
1832         $amount = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1833         $currency = CRM_Utils_Array::retrieveValueRecursive($contribution, 'currency');
1834         $value = CRM_Utils_Money::format($amount, $currency);
1835         break;
1836
1837       case 'receive_date':
1838       case 'receipt_date':
1839         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1840         $config = CRM_Core_Config::singleton();
1841         $value = CRM_Utils_Date::customFormat($value, $config->dateformatDatetime);
1842         break;
1843
1844       case 'source':
1845         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, 'contribution_source');
1846         break;
1847
1848       default:
1849         if (!in_array($token, self::$_tokens['contribution'])) {
1850           $value = "{contribution.$token}";
1851         }
1852         else {
1853           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1854         }
1855         break;
1856     }
1857
1858     if ($escapeSmarty) {
1859       $value = self::tokenEscapeSmarty($value);
1860     }
1861     return $value;
1862   }
1863
1864   /**
1865    * @return array
1866    *   [legacy_token => new_token]
1867    */
1868   public static function legacyContactTokens() {
1869     return [
1870       'individual_prefix' => 'prefix_id',
1871       'individual_suffix' => 'suffix_id',
1872       'gender' => 'gender_id',
1873       'communication_style' => 'communication_style_id',
1874     ];
1875   }
1876
1877   /**
1878    * Get all custom field tokens of $entity
1879    *
1880    * @deprecated
1881    *
1882    * @param string $entity
1883    * @return array
1884    *   return custom field tokens in array('custom_N' => 'label') format
1885    */
1886   public static function getCustomFieldTokens($entity) {
1887     $customTokens = [];
1888     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1889       $customTokens['custom_' . $id] = $info['label'] . ' :: ' . $info['groupTitle'];
1890     }
1891     return $customTokens;
1892   }
1893
1894   /**
1895    * Formats a token list for the select2 widget
1896    *
1897    * @param $tokens
1898    * @return array
1899    */
1900   public static function formatTokensForDisplay($tokens) {
1901     $sorted = $output = [];
1902
1903     // Sort in ascending order by ignoring word case
1904     natcasesort($tokens);
1905
1906     // Attempt to place tokens into optgroups
1907     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1908     foreach ($tokens as $k => $v) {
1909       // Check to see if this token is already in a group e.g. for custom fields
1910       $split = explode(' :: ', $v);
1911       if (!empty($split[1])) {
1912         $sorted[$split[1]][] = ['id' => $k, 'text' => $split[0]];
1913       }
1914       // Group by entity
1915       else {
1916         $split = explode('.', trim($k, '{}'));
1917         if (isset($split[1])) {
1918           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucwords(str_replace('_', ' ', $split[0]));
1919         }
1920         else {
1921           $entity = 'Contact';
1922         }
1923         $sorted[ts($entity)][] = ['id' => $k, 'text' => $v];
1924       }
1925     }
1926
1927     ksort($sorted);
1928     foreach ($sorted as $k => $v) {
1929       $output[] = ['text' => $k, 'children' => $v];
1930     }
1931
1932     return $output;
1933   }
1934
1935   /**
1936    * @param $value
1937    * @param $token
1938    *
1939    * @return bool|int|mixed|string|null
1940    */
1941   protected static function convertPseudoConstantsUsingMetadata($value, $token) {
1942     // Convert pseudoconstants using metadata
1943     if ($value && is_numeric($value)) {
1944       $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
1945       if (!empty($allFields[$token]['pseudoconstant'])) {
1946         $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
1947       }
1948     }
1949     elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
1950       $value = CRM_Utils_Date::customFormat($value);
1951     }
1952     return $value;
1953   }
1954
1955   /**
1956    * Get token deprecation information.
1957    *
1958    * @return array
1959    */
1960   public static function getTokenDeprecations(): array {
1961     return [
1962       'WorkFlowMessageTemplates' => [
1963         'contribution_invoice_receipt' => [
1964           '$display_name' => 'contact.display_name',
1965         ],
1966         'contribution_online_receipt' => [
1967           '$contributeMode' => 'no longer available / relevant',
1968           '$first_name' => 'contact.first_name',
1969           '$last_name' => 'contact.last_name',
1970           '$displayName' => 'contact.display_name',
1971         ],
1972       ],
1973     ];
1974   }
1975
1976 }