CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * Class to abstract token replacement.
  20  */
  21 class CRM_Utils_Token {
  22   public static $_requiredTokens = NULL;
  23
  24   public static $_tokens = [
  25     'action' => [
  26       'forward',
  27       'optOut',
  28       'optOutUrl',
  29       'reply',
  30       'unsubscribe',
  31       'unsubscribeUrl',
  32       'resubscribe',
  33       'resubscribeUrl',
  34       'subscribeUrl',
  35     ],
  36     'mailing' => [
  37       'id',
  38       'key',
  39       'name',
  40       'group',
  41       'subject',
  42       'viewUrl',
  43       'editUrl',
  44       'scheduleUrl',
  45       'approvalStatus',
  46       'approvalNote',
  47       'approveUrl',
  48       'creator',
  49       'creatorEmail',
  50     ],
  51     'user' => [
  52       // we extract the stuff after the role / permission and return the
  53       // civicrm email addresses of all users with that role / permission
  54       // useful with rules integration
  55       'permission:',
  56       'role:',
  57     ],
  58     // populate this dynamically
  59     'contact' => NULL,
  60     // populate this dynamically
  61     'contribution' => NULL,
  62     'domain' => [
  63       'name',
  64       'phone',
  65       'address',
  66       'email',
  67       'id',
  68       'description',
  69     ],
  70     'subscribe' => ['group'],
  71     'unsubscribe' => ['group'],
  72     'resubscribe' => ['group'],
  73     'welcome' => ['group'],
  74   ];
  75
  76   /**
  77    * @deprecated
  78    *   This is used by CiviMail but will be made redundant by FlexMailer.
  79    * @return array
  80    */
  81   public static function getRequiredTokens() {
  82     if (self::$_requiredTokens == NULL) {
  83       self::$_requiredTokens = [
  84         'domain.address' => ts("Domain address - displays your organization's postal address."),
  85         'action.optOutUrl or action.unsubscribeUrl' => [
  86           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
  87           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
  88           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
  89           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
  90         ],
  91       ];
  92     }
  93     return self::$_requiredTokens;
  94   }
  95
  96   /**
  97    * Check a string (mailing body) for required tokens.
  98    *
  99    * @param string $str
 100    *   The message.
 101    *
 102    * @return bool|array
 103    *   true if all required tokens are found,
 104    *    else an array of the missing tokens
 105    */
 106   public static function requiredTokens(&$str) {
 107     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 108     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, []) : CRM_Utils_Token::getRequiredTokens();
 109
 110     $missing = [];
 111     foreach ($requiredTokens as $token => $value) {
 112       if (!is_array($value)) {
 113         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 114           $missing[$token] = $value;
 115         }
 116       }
 117       else {
 118         $present = FALSE;
 119         $desc = NULL;
 120         foreach ($value as $t => $d) {
 121           $desc = $d;
 122           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 123             $present = TRUE;
 124           }
 125         }
 126         if (!$present) {
 127           $missing[$token] = $desc;
 128         }
 129       }
 130     }
 131
 132     if (empty($missing)) {
 133       return TRUE;
 134     }
 135     return $missing;
 136   }
 137
 138   /**
 139    * Wrapper for token matching.
 140    *
 141    * @param string $type
 142    *   The token type (domain,mailing,contact,action).
 143    * @param string $var
 144    *   The token variable.
 145    * @param string $str
 146    *   The string to search.
 147    *
 148    * @return bool
 149    *   Was there a match
 150    */
 151   public static function token_match($type, $var, &$str) {
 152     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 153     return preg_match("/(^|[^\{])$token/", $str);
 154   }
 155
 156   /**
 157    * Wrapper for token replacing.
 158    *
 159    * @param string $type
 160    *   The token type.
 161    * @param string $var
 162    *   The token variable.
 163    * @param string $value
 164    *   The value to substitute for the token.
 165    * @param string $str (reference) The string to replace in
 166    *
 167    * @param bool $escapeSmarty
 168    *
 169    * @return string
 170    *   The processed string
 171    */
 172   public static function token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 173     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 174     if (!$value) {
 175       $value = '$3';
 176     }
 177     if ($escapeSmarty) {
 178       $value = self::tokenEscapeSmarty($value);
 179     }
 180     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 181     return $str;
 182   }
 183
 184   /**
 185    * Get the regex for token replacement
 186    *
 187    * @param string $token_type
 188    *   A string indicating the the type of token to be used in the expression.
 189    *
 190    * @return string
 191    *   regular expression suitable for using in preg_replace
 192    */
 193   private static function tokenRegex(string $token_type) {
 194     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(:|\.)?\w*(\-[\w\s]+)?)\}(?!\})/';
 195   }
 196
 197   /**
 198    * Escape the string so a malicious user cannot inject smarty code into the template.
 199    *
 200    * @param string $string
 201    *   A string that needs to be escaped from smarty parsing.
 202    *
 203    * @return string
 204    *   the escaped string
 205    */
 206   public static function tokenEscapeSmarty($string) {
 207     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 208     return preg_replace(['/{/', '/(?<!{ldelim)}/'], ['{ldelim}', '{rdelim}'], $string);
 209   }
 210
 211   /**
 212    * Replace all the domain-level tokens in $str
 213    *
 214    * @param string $str
 215    *   The string with tokens to be replaced.
 216    * @param object $domain
 217    *   The domain BAO.
 218    * @param bool $html
 219    *   Replace tokens with HTML or plain text.
 220    *
 221    * @param null $knownTokens
 222    * @param bool $escapeSmarty
 223    *
 224    * @return string
 225    *   The processed string
 226    */
 227   public static function replaceDomainTokens(
 228     $str,
 229     $domain,
 230     $html = FALSE,
 231     $knownTokens = NULL,
 232     $escapeSmarty = FALSE
 233   ) {
 234     $key = 'domain';
 235     if (
 236       !$knownTokens || empty($knownTokens[$key])
 237     ) {
 238       return $str;
 239     }
 240
 241     $str = preg_replace_callback(
 242       self::tokenRegex($key),
 243       function ($matches) use ($domain, $html, $escapeSmarty) {
 244         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 245       },
 246       $str
 247     );
 248     return $str;
 249   }
 250
 251   /**
 252    * @param string $token
 253    * @param CRM_Core_BAO_Domain $domain
 254    * @param bool $html
 255    * @param bool $escapeSmarty
 256    *
 257    * @return null|string
 258    */
 259   public static function getDomainTokenReplacement($token, $domain, $html = FALSE, $escapeSmarty = FALSE): ?string {
 260     $tokens = self::getDomainTokens($domain->id, $html);
 261     $value = $tokens[$token] ?? "{domain.$token}";
 262     if ($escapeSmarty) {
 263       $value = self::tokenEscapeSmarty($value);
 264     }
 265     return $value;
 266   }
 267
 268   /**
 269    * Replace all the org-level tokens in $str
 270    *
 271    * @fixme: This function appears to be broken, as it depended on
 272    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 273    * Marking as deprecated until this is clarified.
 274    *
 275    * @deprecated
 276    *  - the above hard-breakage was there from 2015 to 2021 and
 277    * no error was ever reported on it -does that mean
 278    * 1) the code is never hit because the only function that
 279    * calls this function is never called or
 280    * 2) it was called but never required to resolve any tokens
 281    * or more specifically custom field tokens
 282    *
 283    * The handling for custom fields with the removed token has
 284    * now been removed.
 285    *
 286    * @param string $str
 287    *   The string with tokens to be replaced.
 288    * @param object $org
 289    *   Associative array of org properties.
 290    * @param bool $html
 291    *   Replace tokens with HTML or plain text.
 292    *
 293    * @param bool $escapeSmarty
 294    *
 295    * @return string
 296    *   The processed string
 297    */
 298   public static function replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 299     self::$_tokens['org']
 300       = array_merge(
 301         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 302         ['address', 'display_name', 'checksum', 'contact_id']
 303       );
 304
 305     foreach (self::$_tokens['org'] as $token) {
 306       // print "Getting token value for $token<br/><br/>";
 307       if ($token === '') {
 308         continue;
 309       }
 310
 311       // If the string doesn't contain this token, skip it.
 312
 313       if (!self::token_match('org', $token, $str)) {
 314         continue;
 315       }
 316
 317       // Construct value from $token and $contact
 318
 319       $value = NULL;
 320
 321       if ($token === 'checksum') {
 322         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 323         $value = "cs={$cs}";
 324       }
 325       elseif ($token === 'address') {
 326         // Build the location values array
 327
 328         $loc = [];
 329         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 330         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 331         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 332         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 333         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 334
 335         // Construct the address token
 336
 337         $value = CRM_Utils_Address::format($loc);
 338         if ($html) {
 339           $value = str_replace("\n", '<br />', $value);
 340         }
 341       }
 342       else {
 343         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 344       }
 345
 346       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 347     }
 348
 349     return $str;
 350   }
 351
 352   /**
 353    * Replace all mailing tokens in $str
 354    *
 355    * @param string $str
 356    *   The string with tokens to be replaced.
 357    * @param object $mailing
 358    *   The mailing BAO, or null for validation.
 359    * @param bool $html
 360    *   Replace tokens with HTML or plain text.
 361    *
 362    * @param null $knownTokens
 363    * @param bool $escapeSmarty
 364    *
 365    * @return string
 366    *   The processed string
 367    */
 368   public static function &replaceMailingTokens(
 369     $str,
 370     &$mailing,
 371     $html = FALSE,
 372     $knownTokens = NULL,
 373     $escapeSmarty = FALSE
 374   ) {
 375     $key = 'mailing';
 376     if (!$knownTokens || !isset($knownTokens[$key])) {
 377       return $str;
 378     }
 379
 380     $str = preg_replace_callback(
 381       self::tokenRegex($key),
 382       function ($matches) use (&$mailing, $escapeSmarty) {
 383         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 384       },
 385       $str
 386     );
 387     return $str;
 388   }
 389
 390   /**
 391    * @param $token
 392    * @param $mailing
 393    * @param bool $escapeSmarty
 394    *
 395    * @return string
 396    */
 397   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 398     $value = '';
 399     switch ($token) {
 400       // CRM-7663
 401
 402       case 'id':
 403         $value = $mailing ? $mailing->id : 'undefined';
 404         break;
 405
 406       // Key is the ID, or the hash when the hash URLs setting is enabled
 407       case 'key':
 408         $value = $mailing->id;
 409         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($value)) {
 410           $value = $hash;
 411         }
 412         break;
 413
 414       case 'name':
 415         $value = $mailing ? $mailing->name : 'Mailing Name';
 416         break;
 417
 418       case 'group':
 419         $groups = $mailing ? $mailing->getGroupNames() : ['Mailing Groups'];
 420         $value = implode(', ', $groups);
 421         break;
 422
 423       case 'subject':
 424         $value = $mailing->subject;
 425         break;
 426
 427       case 'viewUrl':
 428         $mailingKey = $mailing->id;
 429         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 430           $mailingKey = $hash;
 431         }
 432         $value = CRM_Utils_System::url('civicrm/mailing/view',
 433           "reset=1&id={$mailingKey}",
 434           TRUE, NULL, FALSE, TRUE
 435         );
 436         break;
 437
 438       case 'editUrl':
 439       case 'scheduleUrl':
 440         // Note: editUrl and scheduleUrl used to be different, but now there's
 441         // one screen which can adapt based on permissions (in workflow mode).
 442         $value = CRM_Utils_System::url('civicrm/mailing/send',
 443           "reset=1&mid={$mailing->id}&continue=true",
 444           TRUE, NULL, FALSE, TRUE
 445         );
 446         break;
 447
 448       case 'html':
 449         $page = new CRM_Mailing_Page_View();
 450         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 451         break;
 452
 453       case 'approvalStatus':
 454         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 455         break;
 456
 457       case 'approvalNote':
 458         $value = $mailing->approval_note;
 459         break;
 460
 461       case 'approveUrl':
 462         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 463           "reset=1&mid={$mailing->id}",
 464           TRUE, NULL, FALSE, TRUE
 465         );
 466         break;
 467
 468       case 'creator':
 469         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 470         break;
 471
 472       case 'creatorEmail':
 473         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 474         break;
 475
 476       default:
 477         $value = "{mailing.$token}";
 478         break;
 479     }
 480
 481     if ($escapeSmarty) {
 482       $value = self::tokenEscapeSmarty($value);
 483     }
 484     return $value;
 485   }
 486
 487   /**
 488    * Replace all action tokens in $str
 489    *
 490    * @param string $str
 491    *   The string with tokens to be replaced.
 492    * @param array $addresses
 493    *   Assoc. array of VERP event addresses.
 494    * @param array $urls
 495    *   Assoc. array of action URLs.
 496    * @param bool $html
 497    *   Replace tokens with HTML or plain text.
 498    * @param array $knownTokens
 499    *   A list of tokens that are known to exist in the email body.
 500    *
 501    * @param bool $escapeSmarty
 502    *
 503    * @return string
 504    *   The processed string
 505    */
 506   public static function &replaceActionTokens(
 507     $str,
 508     &$addresses,
 509     &$urls,
 510     $html = FALSE,
 511     $knownTokens = NULL,
 512     $escapeSmarty = FALSE
 513   ) {
 514     $key = 'action';
 515     // here we intersect with the list of pre-configured valid tokens
 516     // so that we remove anything we do not recognize
 517     // I hope to move this step out of here soon and
 518     // then we will just iterate on a list of tokens that are passed to us
 519     if (!$knownTokens || empty($knownTokens[$key])) {
 520       return $str;
 521     }
 522
 523     $str = preg_replace_callback(
 524       self::tokenRegex($key),
 525       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 526         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 527       },
 528       $str
 529     );
 530     return $str;
 531   }
 532
 533   /**
 534    * @param $token
 535    * @param $addresses
 536    * @param $urls
 537    * @param bool $html
 538    * @param bool $escapeSmarty
 539    *
 540    * @return mixed|string
 541    */
 542   public static function getActionTokenReplacement(
 543     $token,
 544     &$addresses,
 545     &$urls,
 546     $html = FALSE,
 547     $escapeSmarty = FALSE
 548   ) {
 549     // If the token is an email action, use it.  Otherwise, find the
 550     // appropriate URL.
 551
 552     if (!in_array($token, self::$_tokens['action'])) {
 553       $value = "{action.$token}";
 554     }
 555     else {
 556       $value = $addresses[$token] ?? NULL;
 557
 558       if ($value == NULL) {
 559         $value = $urls[$token] ?? NULL;
 560       }
 561
 562       if ($value && $html) {
 563         // fix for CRM-2318
 564         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 565           $value = "mailto:$value";
 566         }
 567       }
 568       elseif ($value && !$html) {
 569         $value = str_replace('&amp;', '&', $value);
 570       }
 571     }
 572
 573     if ($escapeSmarty) {
 574       $value = self::tokenEscapeSmarty($value);
 575     }
 576     return $value;
 577   }
 578
 579   /**
 580    * Replace all the contact-level tokens in $str with information from
 581    * $contact.
 582    *
 583    * @param string $str
 584    *   The string with tokens to be replaced.
 585    * @param array $contact
 586    *   Associative array of contact properties.
 587    * @param bool $html
 588    *   Replace tokens with HTML or plain text.
 589    * @param array $knownTokens
 590    *   A list of tokens that are known to exist in the email body.
 591    * @param bool $returnBlankToken
 592    *   Return unevaluated token if value is null.
 593    *
 594    * @param bool $escapeSmarty
 595    *
 596    * @return string
 597    *   The processed string
 598    */
 599   public static function replaceContactTokens(
 600     $str,
 601     &$contact,
 602     $html = FALSE,
 603     $knownTokens = NULL,
 604     $returnBlankToken = FALSE,
 605     $escapeSmarty = FALSE
 606   ) {
 607     // Refresh contact tokens in case they have changed. There is heavy caching
 608     // in exportable fields so there is no benefit in doing this conditionally.
 609     self::$_tokens['contact'] = array_merge(
 610       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 611       ['checksum', 'contact_id']
 612     );
 613
 614     $key = 'contact';
 615     // here we intersect with the list of pre-configured valid tokens
 616     // so that we remove anything we do not recognize
 617     // I hope to move this step out of here soon and
 618     // then we will just iterate on a list of tokens that are passed to us
 619     if (!$knownTokens || empty($knownTokens[$key])) {
 620       return $str;
 621     }
 622
 623     $str = preg_replace_callback(
 624       self::tokenRegex($key),
 625       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 626         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 627       },
 628       $str
 629     );
 630
 631     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 632     return $str;
 633   }
 634
 635   /**
 636    * @param $token
 637    * @param $contact
 638    * @param bool $html
 639    * @param bool $returnBlankToken
 640    * @param bool $escapeSmarty
 641    *
 642    * @return bool|mixed|null|string
 643    */
 644   public static function getContactTokenReplacement(
 645     $token,
 646     &$contact,
 647     $html = FALSE,
 648     $returnBlankToken = FALSE,
 649     $escapeSmarty = FALSE
 650   ) {
 651     if (self::$_tokens['contact'] == NULL) {
 652       /* This should come from UF */
 653
 654       self::$_tokens['contact']
 655         = array_merge(
 656           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 657           ['checksum', 'contact_id']
 658         );
 659     }
 660
 661     // Construct value from $token and $contact
 662
 663     $value = NULL;
 664     $noReplace = FALSE;
 665
 666     // Support legacy tokens
 667     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 668
 669     // check if the token we were passed is valid
 670     // we have to do this because this function is
 671     // called only when we find a token in the string
 672
 673     if (!in_array($token, self::$_tokens['contact'])) {
 674       $noReplace = TRUE;
 675     }
 676     elseif ($token == 'checksum') {
 677       $hash = $contact['hash'] ?? NULL;
 678       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 679       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 680         NULL,
 681         NULL,
 682         $hash
 683       );
 684       $value = "cs={$cs}";
 685     }
 686     else {
 687       $value = (array) CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 688
 689       foreach ($value as $index => $item) {
 690         $value[$index] = self::convertPseudoConstantsUsingMetadata($value[$index], $token);
 691       }
 692       $value = implode(', ', $value);
 693     }
 694
 695     if (!$html) {
 696       $value = str_replace('&amp;', '&', $value);
 697     }
 698
 699     // if null then return actual token
 700     if ($returnBlankToken && !$value) {
 701       $noReplace = TRUE;
 702     }
 703
 704     if ($noReplace) {
 705       $value = "{contact.$token}";
 706     }
 707
 708     if ($escapeSmarty
 709       && !($returnBlankToken && $noReplace)
 710     ) {
 711       // $returnBlankToken means the caller wants to do further attempts at
 712       // processing unreplaced tokens -- so don't escape them yet in this case.
 713       $value = self::tokenEscapeSmarty($value);
 714     }
 715
 716     return $value;
 717   }
 718
 719   /**
 720    * Replace all the hook tokens in $str with information from
 721    * $contact.
 722    *
 723    * @param string $str
 724    *   The string with tokens to be replaced.
 725    * @param array $contact
 726    *   Associative array of contact properties (including hook token values).
 727    * @param $categories
 728    * @param bool $html
 729    *   Replace tokens with HTML or plain text.
 730    *
 731    * @param bool $escapeSmarty
 732    *
 733    * @return string
 734    *   The processed string
 735    */
 736   public static function &replaceHookTokens(
 737     $str,
 738     &$contact,
 739     $categories = NULL,
 740     $html = FALSE,
 741     $escapeSmarty = FALSE
 742   ) {
 743     if (!$categories) {
 744       $categories = self::getTokenCategories();
 745     }
 746     foreach ($categories as $key) {
 747       $str = preg_replace_callback(
 748         self::tokenRegex($key),
 749         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 750           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 751         },
 752         $str
 753       );
 754     }
 755     return $str;
 756   }
 757
 758   /**
 759    * Get the categories required for rendering tokens.
 760    *
 761    * @return array
 762    */
 763   public static function getTokenCategories(): array {
 764     if (!isset(\Civi::$statics[__CLASS__]['token_categories'])) {
 765       $tokens = [];
 766       \CRM_Utils_Hook::tokens($tokens);
 767       \Civi::$statics[__CLASS__]['token_categories'] = array_keys($tokens);
 768     }
 769     return \Civi::$statics[__CLASS__]['token_categories'];
 770   }
 771
 772   /**
 773    * Parse html through Smarty resolving any smarty functions.
 774    * @param string $tokenHtml
 775    * @param array $entity
 776    * @param string $entityType
 777    * @return string
 778    *   html parsed through smarty
 779    */
 780   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 781     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 782       $smarty = CRM_Core_Smarty::singleton();
 783       // also add the tokens to the template
 784       $smarty->assign_by_ref($entityType, $entity);
 785       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 786     }
 787     return $tokenHtml;
 788   }
 789
 790   /**
 791    * @param $token
 792    * @param $contact
 793    * @param $category
 794    * @param bool $html
 795    * @param bool $escapeSmarty
 796    *
 797    * @return mixed|string
 798    */
 799   public static function getHookTokenReplacement(
 800     $token,
 801     &$contact,
 802     $category,
 803     $html = FALSE,
 804     $escapeSmarty = FALSE
 805   ) {
 806     $value = $contact["{$category}.{$token}"] ?? NULL;
 807
 808     if ($value && !$html) {
 809       $value = str_replace('&amp;', '&', $value);
 810     }
 811
 812     if ($escapeSmarty) {
 813       $value = self::tokenEscapeSmarty($value);
 814     }
 815
 816     return $value;
 817   }
 818
 819   /**
 820    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 821    * for example {{token}}  or \{token}  will result in {token} in the final email
 822    *
 823    * this routine will remove the extra backslashes and braces
 824    *
 825    * @param $str ref to the string that will be scanned and modified
 826    */
 827   public static function unescapeTokens(&$str) {
 828     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 829   }
 830
 831   /**
 832    * Replace unsubscribe tokens.
 833    *
 834    * @param string $str
 835    *   The string with tokens to be replaced.
 836    * @param object $domain
 837    *   The domain BAO.
 838    * @param array $groups
 839    *   The groups (if any) being unsubscribed.
 840    * @param bool $html
 841    *   Replace tokens with html or plain text.
 842    * @param int $contact_id
 843    *   The contact ID.
 844    * @param string $hash The security hash of the unsub event
 845    *
 846    * @return string
 847    *   The processed string
 848    */
 849   public static function &replaceUnsubscribeTokens(
 850     $str,
 851     &$domain,
 852     &$groups,
 853     $html,
 854     $contact_id,
 855     $hash
 856   ) {
 857     if (self::token_match('unsubscribe', 'group', $str)) {
 858       if (!empty($groups)) {
 859         $config = CRM_Core_Config::singleton();
 860         $base = CRM_Utils_System::baseURL();
 861
 862         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 863         $dao = new CRM_Contact_DAO_Group();
 864         $dao->find();
 865         while ($dao->fetch()) {
 866           if (substr($dao->visibility, 0, 6) == 'Public') {
 867             $visibleGroups[] = $dao->id;
 868           }
 869         }
 870         $value = implode(', ', $groups);
 871         self::token_replace('unsubscribe', 'group', $value, $str);
 872       }
 873     }
 874     return $str;
 875   }
 876
 877   /**
 878    * Replace resubscribe tokens.
 879    *
 880    * @param string $str
 881    *   The string with tokens to be replaced.
 882    * @param object $domain
 883    *   The domain BAO.
 884    * @param array $groups
 885    *   The groups (if any) being resubscribed.
 886    * @param bool $html
 887    *   Replace tokens with html or plain text.
 888    * @param int $contact_id
 889    *   The contact ID.
 890    * @param string $hash The security hash of the resub event
 891    *
 892    * @return string
 893    *   The processed string
 894    */
 895   public static function &replaceResubscribeTokens(
 896     $str, &$domain, &$groups, $html,
 897     $contact_id, $hash
 898   ) {
 899     if (self::token_match('resubscribe', 'group', $str)) {
 900       if (!empty($groups)) {
 901         $value = implode(', ', $groups);
 902         self::token_replace('resubscribe', 'group', $value, $str);
 903       }
 904     }
 905     return $str;
 906   }
 907
 908   /**
 909    * Replace subscription-confirmation-request tokens
 910    *
 911    * @param string $str
 912    *   The string with tokens to be replaced.
 913    * @param string $group
 914    *   The name of the group being subscribed.
 915    * @param $url
 916    * @param bool $html
 917    *   Replace tokens with html or plain text.
 918    *
 919    * @return string
 920    *   The processed string
 921    */
 922   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 923     if (self::token_match('subscribe', 'group', $str)) {
 924       self::token_replace('subscribe', 'group', $group, $str);
 925     }
 926     if (self::token_match('subscribe', 'url', $str)) {
 927       self::token_replace('subscribe', 'url', $url, $str);
 928     }
 929     return $str;
 930   }
 931
 932   /**
 933    * Replace subscription-invitation tokens
 934    *
 935    * @param string $str
 936    *   The string with tokens to be replaced.
 937    *
 938    * @return string
 939    *   The processed string
 940    */
 941   public static function &replaceSubscribeInviteTokens($str) {
 942     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 943       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 944         'reset=1',
 945         TRUE, NULL, FALSE, TRUE
 946       );
 947       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 948     }
 949
 950     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 951       foreach ($matches as $key => $value) {
 952         $gid = substr($value, 21, -1);
 953         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 954           "reset=1&gid={$gid}",
 955           TRUE, NULL, FALSE, TRUE
 956         );
 957         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
 958       }
 959     }
 960
 961     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
 962       foreach ($matches as $key => $value) {
 963         $gid = substr($value, 18, -1);
 964         $config = CRM_Core_Config::singleton();
 965         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
 966         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
 967         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
 968         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
 969       }
 970     }
 971     return $str;
 972   }
 973
 974   /**
 975    * Replace welcome/confirmation tokens
 976    *
 977    * @param string $str
 978    *   The string with tokens to be replaced.
 979    * @param string $group
 980    *   The name of the group being subscribed.
 981    * @param bool $html
 982    *   Replace tokens with html or plain text.
 983    *
 984    * @return string
 985    *   The processed string
 986    */
 987   public static function &replaceWelcomeTokens($str, $group, $html) {
 988     if (self::token_match('welcome', 'group', $str)) {
 989       self::token_replace('welcome', 'group', $group, $str);
 990     }
 991     return $str;
 992   }
 993
 994   /**
 995    * Find unprocessed tokens (call this last)
 996    *
 997    * @param string $str
 998    *   The string to search.
 999    *
1000    * @return array
1001    *   Array of tokens that weren't replaced
1002    */
1003   public static function &unmatchedTokens(&$str) {
1004     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1005     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1006     return $match[1];
1007   }
1008
1009   /**
1010    * Find and replace tokens for each component.
1011    *
1012    * @param string $str
1013    *   The string to search.
1014    * @param array $contact
1015    *   Associative array of contact properties.
1016    * @param array $components
1017    *   A list of tokens that are known to exist in the email body.
1018    *
1019    * @param bool $escapeSmarty
1020    * @param bool $returnEmptyToken
1021    *
1022    * @return string
1023    *   The processed string
1024    */
1025   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1026     if (!is_array($components) || empty($contact)) {
1027       return $str;
1028     }
1029
1030     foreach ($components as $name => $tokens) {
1031       if (!is_array($tokens) || empty($tokens)) {
1032         continue;
1033       }
1034
1035       foreach ($tokens as $token) {
1036         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1037           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1038         }
1039         elseif (!$returnEmptyToken) {
1040           //replacing empty token
1041           self::token_replace($name, $token, "", $str, $escapeSmarty);
1042         }
1043       }
1044     }
1045     return $str;
1046   }
1047
1048   /**
1049    * Get array of string tokens.
1050    *
1051    * @param string $string
1052    *   The input string to parse for tokens.
1053    *
1054    * @return array
1055    *   array of tokens mentioned in field
1056    */
1057   public static function getTokens($string) {
1058     $matches = [];
1059     $tokens = [];
1060     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+(:|.)?\w*)\}(?!\})/',
1061       $string,
1062       $matches,
1063       PREG_PATTERN_ORDER
1064     );
1065
1066     if ($matches[1]) {
1067       foreach ($matches[1] as $token) {
1068         $parts = explode('.', $token, 3);
1069         $type = $parts[0];
1070         $name = $parts[1];
1071         $suffix = !empty($parts[2]) ? ('.' . $parts[2]) : '';
1072         if ($name && $type) {
1073           if (!isset($tokens[$type])) {
1074             $tokens[$type] = [];
1075           }
1076           $tokens[$type][] = $name . $suffix;
1077         }
1078       }
1079     }
1080     return $tokens;
1081   }
1082
1083   /**
1084    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1085    * and getTokens appears to be historical rather than intentional and should be reviewed
1086    * @param $string
1087    * @return array
1088    *   fields to pass in as return properties when populating token
1089    */
1090   public static function getReturnProperties(&$string) {
1091     $returnProperties = [];
1092     $matches = [];
1093     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1094       $string,
1095       $matches,
1096       PREG_PATTERN_ORDER
1097     );
1098     if ($matches[1]) {
1099       foreach ($matches[1] as $token) {
1100         [$type, $name] = preg_split('/\./', $token, 2);
1101         if ($name) {
1102           $returnProperties["{$name}"] = 1;
1103         }
1104       }
1105     }
1106
1107     return $returnProperties;
1108   }
1109
1110   /**
1111    * Gives required details of contacts in an indexed array format so we
1112    * can iterate in a nice loop and do token evaluation
1113    *
1114    * @param array $contactIDs
1115    * @param array $returnProperties
1116    *   Of required properties.
1117    * @param bool $skipOnHold Don't return on_hold contact info also.
1118    *   Don't return on_hold contact info also.
1119    * @param bool $skipDeceased Don't return deceased contact info.
1120    *   Don't return deceased contact info.
1121    * @param array $extraParams
1122    *   Extra params - DEPRECATED
1123    * @param array $tokens
1124    *   The list of tokens we've extracted from the content.
1125    * @param string|null $className
1126    * @param int|null $jobID
1127    *   The mailing list jobID - this is a legacy param.
1128    *
1129    * @return array - e.g [[1 => ['first_name' => 'bob'...], 34 => ['first_name' => 'fred'...]]]
1130    */
1131   public static function getTokenDetails(
1132     $contactIDs,
1133     $returnProperties = NULL,
1134     $skipOnHold = TRUE,
1135     $skipDeceased = TRUE,
1136     $extraParams = NULL,
1137     $tokens = [],
1138     $className = NULL,
1139     $jobID = NULL
1140   ) {
1141
1142     $params = [];
1143     foreach ($contactIDs as $contactID) {
1144       $params[] = [
1145         CRM_Core_Form::CB_PREFIX . $contactID,
1146         '=',
1147         1,
1148         0,
1149         0,
1150       ];
1151     }
1152
1153     // fix for CRM-2613
1154     if ($skipDeceased) {
1155       $params[] = ['is_deceased', '=', 0, 0, 0];
1156     }
1157
1158     //fix for CRM-3798
1159     if ($skipOnHold) {
1160       $params[] = ['on_hold', '=', 0, 0, 0];
1161     }
1162
1163     if ($extraParams) {
1164       CRM_Core_Error::deprecatedWarning('Passing $extraParams to getTokenDetails() is not supported and will be removed in a future version');
1165       $params = array_merge($params, $extraParams);
1166     }
1167
1168     // if return properties are not passed then get all return properties
1169     if (empty($returnProperties)) {
1170       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1171         ['display_name', 'checksum', 'contact_id']
1172       );
1173       foreach ($fields as $val) {
1174         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1175         // with contacts and are expensive to resolve.
1176         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1177         // and upstream calls to this should populate return properties.
1178         $unavailableFields = ['group', 'tag'];
1179         if (!in_array($val, $unavailableFields)) {
1180           $returnProperties[$val] = 1;
1181         }
1182       }
1183     }
1184
1185     $custom = [];
1186     foreach ($returnProperties as $name => $dontCare) {
1187       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1188       if ($cfID) {
1189         $custom[] = $cfID;
1190       }
1191     }
1192
1193     [$contactDetails] = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1194
1195     foreach ($contactIDs as $contactID) {
1196       if (array_key_exists($contactID, $contactDetails)) {
1197         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1198         ) {
1199           $communicationPreferences = [];
1200           foreach ((array) $contactDetails[$contactID]['preferred_communication_method'] as $val) {
1201             if ($val) {
1202               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1203             }
1204           }
1205           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1206         }
1207
1208         foreach ($custom as $cfID) {
1209           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1210             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1211           }
1212         }
1213
1214         // special case for greeting replacement
1215         foreach ([
1216           'email_greeting',
1217           'postal_greeting',
1218           'addressee',
1219         ] as $val) {
1220           if (!empty($contactDetails[$contactID][$val])) {
1221             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1222           }
1223         }
1224       }
1225     }
1226
1227     // $contactDetails = &$details[0] = is an array of [ contactID => contactDetails ]
1228     // also call a hook and get token details
1229     CRM_Utils_Hook::tokenValues($contactDetails,
1230       $contactIDs,
1231       $jobID,
1232       $tokens,
1233       $className
1234     );
1235     return [$contactDetails];
1236   }
1237
1238   /**
1239    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1240    * specific tokens to be rendered
1241    *
1242    * @param array $contactIDs
1243    *   This should always be array(0) or its not anonymous - left to keep signature same.
1244    *   as main fn
1245    * @param string $returnProperties
1246    * @param bool $skipOnHold
1247    * @param bool $skipDeceased
1248    * @param string $extraParams
1249    * @param array $tokens
1250    * @param string $className
1251    *   Sent as context to the hook.
1252    * @param string $jobID
1253    * @return array
1254    *   contactDetails with hooks swapped out
1255    */
1256   public static function getAnonymousTokenDetails($contactIDs = [0],
1257                                            $returnProperties = NULL,
1258                                            $skipOnHold = TRUE,
1259                                            $skipDeceased = TRUE,
1260                                            $extraParams = NULL,
1261                                            $tokens = [],
1262                                            $className = NULL,
1263                                            $jobID = NULL) {
1264     $details = [0 => []];
1265     // also call a hook and get token details
1266     CRM_Utils_Hook::tokenValues($details[0],
1267       $contactIDs,
1268       $jobID,
1269       $tokens,
1270       $className
1271     );
1272     return $details;
1273   }
1274
1275   /**
1276    * Get Membership Token Details.
1277    * @param array $membershipIDs
1278    *   Array of membership IDS.
1279    */
1280   public static function getMembershipTokenDetails($membershipIDs) {
1281     $memberships = civicrm_api3('membership', 'get', [
1282       'options' => ['limit' => 0],
1283       'membership_id' => ['IN' => (array) $membershipIDs],
1284     ]);
1285     return $memberships['values'];
1286   }
1287
1288   /**
1289    * Replace existing greeting tokens in message/subject.
1290    *
1291    * This function operates by reference, modifying the first parameter. Other
1292    * methods for token replacement in this class return the modified string.
1293    * This leads to inconsistency in how these methods must be applied.
1294    *
1295    * @TODO Remove that inconsistency in usage.
1296    *
1297    * ::replaceContactTokens() may need to be called after this method, to
1298    * replace tokens supplied from this method.
1299    *
1300    * @param string $tokenString
1301    * @param array $contactDetails
1302    * @param int $contactId
1303    * @param string $className
1304    * @param bool $escapeSmarty
1305    */
1306   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1307
1308     if (!$contactDetails && !$contactId) {
1309       return;
1310     }
1311
1312     // check if there are any tokens
1313     $greetingTokens = self::getTokens($tokenString);
1314
1315     if (!empty($greetingTokens)) {
1316       // first use the existing contact object for token replacement
1317       if (!empty($contactDetails)) {
1318         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1319       }
1320
1321       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1322       // check if there are any unevaluated tokens
1323       $greetingTokens = self::getTokens($tokenString);
1324
1325       // $greetingTokens not empty, means there are few tokens which are not
1326       // evaluated, like custom data etc
1327       // so retrieve it from database
1328       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1329         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1330         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1331         $contactParams = ['contact_id' => $contactId];
1332
1333         $greetingDetails = self::getTokenDetails($contactParams,
1334           $greetingsReturnProperties,
1335           FALSE, FALSE, NULL,
1336           $greetingTokens,
1337           $className
1338         );
1339
1340         // again replace tokens
1341         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1342           $greetingDetails,
1343           TRUE,
1344           $greetingTokens,
1345           TRUE,
1346           $escapeSmarty
1347         );
1348       }
1349
1350       // check if there are still any unevaluated tokens
1351       $remainingTokens = self::getTokens($tokenString);
1352
1353       // $greetingTokens not empty, there are customized or hook tokens to replace
1354       if (!empty($remainingTokens)) {
1355         // Fill the return properties array
1356         $greetingTokens = $remainingTokens;
1357         reset($greetingTokens);
1358         $greetingsReturnProperties = [];
1359         foreach ($greetingTokens as $value) {
1360           $props = array_flip($value);
1361           $props = array_fill_keys(array_keys($props), 1);
1362           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1363         }
1364         $contactParams = ['contact_id' => $contactId];
1365         $greetingDetails = self::getTokenDetails($contactParams,
1366           $greetingsReturnProperties,
1367           FALSE, FALSE, NULL,
1368           $greetingTokens,
1369           $className
1370         );
1371         // Prepare variables for calling replaceHookTokens
1372         $categories = array_keys($greetingTokens);
1373         [$contact] = $greetingDetails;
1374         // Replace tokens defined in Hooks.
1375         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1376       }
1377     }
1378   }
1379
1380   /**
1381    * At this point, $contactDetails has loaded the contact from the DAO. Any
1382    * (non-custom) missing fields are null.  By removing them, we can avoid
1383    * expensive calls to CRM_Contact_BAO_Query.
1384    *
1385    * @param string $tokenString
1386    * @param array $contactDetails
1387    * @param array $greetingTokens
1388    */
1389   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1390
1391     // Only applies to contact tokens
1392     if (!array_key_exists('contact', $greetingTokens)) {
1393       return;
1394     }
1395
1396     $greetingTokensOriginal = $greetingTokens;
1397     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1398     // Sometimes contactDetails are in a multidemensional array, sometimes a
1399     // single-dimension array.
1400     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1401       $contactDetails = current($contactDetails[0]);
1402     }
1403     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1404
1405     // Handle legacy tokens
1406     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1407       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1408         $nullFields[] = $oldToken;
1409       }
1410     }
1411
1412     // Remove null contact fields from $greetingTokens
1413     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1414
1415     // Also remove them from $tokenString
1416     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1417     // Handle legacy tokens again, sigh
1418     if (!empty($removedTokens)) {
1419       foreach ($removedTokens as $token) {
1420         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1421           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1422         }
1423       }
1424       foreach ($removedTokens as $token) {
1425         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1426       }
1427     }
1428   }
1429
1430   /**
1431    * @param $tokens
1432    *
1433    * @return array
1434    */
1435   public static function flattenTokens(&$tokens) {
1436     $flattenTokens = [];
1437
1438     foreach ([
1439       'html',
1440       'text',
1441       'subject',
1442     ] as $prop) {
1443       if (!isset($tokens[$prop])) {
1444         continue;
1445       }
1446       foreach ($tokens[$prop] as $type => $names) {
1447         if (!isset($flattenTokens[$type])) {
1448           $flattenTokens[$type] = [];
1449         }
1450         foreach ($names as $name) {
1451           $flattenTokens[$type][$name] = 1;
1452         }
1453       }
1454     }
1455
1456     return $flattenTokens;
1457   }
1458
1459   /**
1460    * Replace all user tokens in $str
1461    *
1462    * @param string $str
1463    *   The string with tokens to be replaced.
1464    *
1465    * @param null $knownTokens
1466    * @param bool $escapeSmarty
1467    *
1468    * @return string
1469    *   The processed string
1470    */
1471   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1472     $key = 'user';
1473     if (!$knownTokens ||
1474       !isset($knownTokens[$key])
1475     ) {
1476       return $str;
1477     }
1478
1479     $str = preg_replace_callback(
1480       self::tokenRegex($key),
1481       function ($matches) use ($escapeSmarty) {
1482         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1483       },
1484       $str
1485     );
1486     return $str;
1487   }
1488
1489   /**
1490    * @param $token
1491    * @param bool $escapeSmarty
1492    *
1493    * @return string
1494    */
1495   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1496     $value = '';
1497
1498     [$objectName, $objectValue] = explode('-', $token, 2);
1499
1500     switch ($objectName) {
1501       case 'permission':
1502         $value = CRM_Core_Permission::permissionEmails($objectValue);
1503         break;
1504
1505       case 'role':
1506         $value = CRM_Core_Permission::roleEmails($objectValue);
1507         break;
1508     }
1509
1510     if ($escapeSmarty) {
1511       $value = self::tokenEscapeSmarty($value);
1512     }
1513
1514     return $value;
1515   }
1516
1517   protected static function _buildContributionTokens() {
1518     $key = 'contribution';
1519
1520     if (!isset(Civi::$statics[__CLASS__][__FUNCTION__][$key])) {
1521       $processor = new CRM_Contribute_Tokens();
1522       $tokens = array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1523         ['campaign' => [], 'financial_type' => [], 'payment_instrument' => []],
1524         self::getCustomFieldTokens('Contribution'),
1525         $processor->getPseudoTokens()
1526       );
1527       foreach ($tokens as $token) {
1528         if (!empty($token['name'])) {
1529           $tokens[$token['name']] = [];
1530         }
1531       }
1532       Civi::$statics[__CLASS__][__FUNCTION__][$key] = array_keys($tokens);
1533     }
1534     self::$_tokens[$key] = Civi::$statics[__CLASS__][__FUNCTION__][$key];
1535   }
1536
1537   /**
1538    * Replace tokens for an entity.
1539    * @param string $entity
1540    * @param array $entityArray
1541    *   (e.g. in format from api).
1542    * @param string $str
1543    *   String to replace in.
1544    * @param array $knownTokens
1545    *   Array of tokens present.
1546    * @param bool $escapeSmarty
1547    * @return string
1548    *   string with replacements made
1549    */
1550   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1551     if (!$knownTokens || empty($knownTokens[$entity])) {
1552       return $str;
1553     }
1554
1555     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1556     $fn = is_callable(['CRM_Utils_Token', $fn]) ? $fn : 'getApiTokenReplacement';
1557     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1558     foreach ($knownTokens[$entity] as $token) {
1559       // We are now supporting the syntax case_type_id:label
1560       // so strip anything after the ':'
1561       // (we aren't supporting 'name' at this stage, so we can assume 'label'
1562       // test cover in TokenConsistencyTest.
1563       $parts = explode(':', $token);
1564       $replacement = self::$fn($entity, $parts[0], $entityArray);
1565       if ($escapeSmarty) {
1566         $replacement = self::tokenEscapeSmarty($replacement);
1567       }
1568       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1569     }
1570     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1571   }
1572
1573   /**
1574    * @param int $caseId
1575    * @param string $str
1576    * @param array $knownTokens
1577    * @param bool $escapeSmarty
1578    * @return string
1579    * @throws \CiviCRM_API3_Exception
1580    */
1581   public static function replaceCaseTokens($caseId, $str, $knownTokens = NULL, $escapeSmarty = FALSE): string {
1582     if (strpos($str, '{case.') === FALSE) {
1583       return $str;
1584     }
1585     if (!$knownTokens) {
1586       $knownTokens = self::getTokens($str);
1587     }
1588     $case = civicrm_api3('case', 'getsingle', ['id' => $caseId]);
1589     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1590   }
1591
1592   /**
1593    * Generic function for formatting token replacement for an api field
1594    *
1595    * @param string $entity
1596    * @param string $token
1597    * @param array $entityArray
1598    * @return string
1599    * @throws \CiviCRM_API3_Exception
1600    */
1601   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1602     if (!isset($entityArray[$token])) {
1603       return '';
1604     }
1605     $field = civicrm_api3($entity, 'getfield', ['action' => 'get', 'name' => $token, 'get_options' => 'get']);
1606     $field = $field['values'];
1607     $fieldType = $field['type'] ?? NULL;
1608     // Boolean fields
1609     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1610       $field['options'] = [ts('No'), ts('Yes')];
1611     }
1612     // Match pseudoconstants
1613     if (!empty($field['options'])) {
1614       $ret = [];
1615       foreach ((array) $entityArray[$token] as $val) {
1616         $ret[] = $field['options'][$val];
1617       }
1618       return implode(', ', $ret);
1619     }
1620     // Format date fields
1621     elseif ($entityArray[$token] && in_array($fieldType, [CRM_Utils_Type::T_DATE, CRM_Utils_Type::T_TIMESTAMP, (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME)])) {
1622       return CRM_Utils_Date::customFormat($entityArray[$token]);
1623     }
1624     return implode(', ', (array) $entityArray[$token]);
1625   }
1626
1627   /**
1628    * Replace Contribution tokens in html.
1629    *
1630    * @param string $str
1631    * @param array $contribution
1632    * @param bool|string $html
1633    * @param string $knownTokens
1634    * @param bool|string $escapeSmarty
1635    *
1636    * @return mixed
1637    */
1638   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1639     $key = 'contribution';
1640     if (!$knownTokens || empty($knownTokens[$key])) {
1641       //early return
1642       return $str;
1643     }
1644
1645     // here we intersect with the list of pre-configured valid tokens
1646     // so that we remove anything we do not recognize
1647     // I hope to move this step out of here soon and
1648     // then we will just iterate on a list of tokens that are passed to us
1649
1650     $str = preg_replace_callback(
1651       self::tokenRegex($key),
1652       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1653         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1654       },
1655       $str
1656     );
1657
1658     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1659     return $str;
1660   }
1661
1662   /**
1663    * We have a situation where we are rendering more than one token in each field because we are combining
1664    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1665    *
1666    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1667    * it applies (other tokens are OK including date fields)
1668    *
1669    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1670    * does - but the fields is already being formatted through a date function
1671    *
1672    * @param string $separator
1673    * @param string $str
1674    * @param array $contributions
1675    * @param array $knownTokens
1676    *
1677    * @return string
1678    */
1679   public static function replaceMultipleContributionTokens(string $separator, string $str, array $contributions, array $knownTokens): string {
1680     foreach ($knownTokens['contribution'] ?? [] as $token) {
1681       $resolvedTokens = [];
1682       foreach ($contributions as $contribution) {
1683         $resolvedTokens[] = self::replaceContributionTokens('{contribution.' . $token . '}', $contribution, FALSE, $knownTokens);
1684       }
1685       $str = self::token_replace('contribution', $token, implode($separator, $resolvedTokens), $str);
1686     }
1687     return $str;
1688   }
1689
1690   /**
1691    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1692    * - this is used by the pdfLetter task from membership search
1693    *
1694    * This is called via replaceEntityTokens.
1695    *
1696    * In the near term it will not be called at all from core as
1697    * the pdf letter task is updated to use the processor.
1698    *
1699    * @deprecated
1700    *
1701    * @param string $entity
1702    *   should always be "membership"
1703    * @param string $token
1704    *   field name
1705    * @param array $membership
1706    *   An api result array for a single membership.
1707    * @return string token replacement
1708    */
1709   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1710     $supportedTokens = [
1711       'id',
1712       'status',
1713       'status_id',
1714       'type',
1715       'membership_type_id',
1716       'start_date',
1717       'join_date',
1718       'end_date',
1719       'fee',
1720     ];
1721     switch ($token) {
1722       case 'type':
1723         // membership_type_id would only be requested if the calling
1724         // class is mapping it to '{membership:membership_type_id:label'}
1725       case 'membership_type_id':
1726         $value = $membership['membership_name'];
1727         break;
1728
1729       case 'status':
1730         // status_id would only be requested if the calling
1731         // class is mapping it to '{membership:status_id:label'}
1732       case 'status_id':
1733         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1734         $value = $statuses[$membership['status_id']];
1735         break;
1736
1737       case 'fee':
1738         try {
1739           $value = civicrm_api3('membership_type', 'getvalue', [
1740             'id' => $membership['membership_type_id'],
1741             'return' => 'minimum_fee',
1742           ]);
1743           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1744         }
1745         catch (CiviCRM_API3_Exception $e) {
1746           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1747           // api handles NULL (4.4)
1748           $value = 0;
1749         }
1750         break;
1751
1752       default:
1753         if (in_array($token, $supportedTokens)) {
1754           $value = $membership[$token];
1755           if (CRM_Utils_String::endsWith($token, '_date')) {
1756             $value = CRM_Utils_Date::customFormat($value);
1757           }
1758         }
1759         else {
1760           // ie unchanged
1761           $value = "{$entity}.{$token}";
1762         }
1763         break;
1764     }
1765
1766     return $value;
1767   }
1768
1769   /**
1770    * @param $token
1771    * @param $contribution
1772    * @param bool $html
1773    * @param bool $escapeSmarty
1774    *
1775    * @return mixed|string
1776    * @throws \CRM_Core_Exception
1777    */
1778   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1779     self::_buildContributionTokens();
1780
1781     switch ($token) {
1782       case 'total_amount':
1783       case 'net_amount':
1784       case 'fee_amount':
1785       case 'non_deductible_amount':
1786         // FIXME: Is this ever a multi-dimensional array?  Why use retrieveValueRecursive()?
1787         $amount = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1788         $currency = CRM_Utils_Array::retrieveValueRecursive($contribution, 'currency');
1789         $value = CRM_Utils_Money::format($amount, $currency);
1790         break;
1791
1792       case 'receive_date':
1793       case 'receipt_date':
1794         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1795         $config = CRM_Core_Config::singleton();
1796         $value = CRM_Utils_Date::customFormat($value, $config->dateformatDatetime);
1797         break;
1798
1799       default:
1800         if (!in_array($token, self::$_tokens['contribution'])) {
1801           $value = "{contribution.$token}";
1802         }
1803         else {
1804           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1805         }
1806         break;
1807     }
1808
1809     if ($escapeSmarty) {
1810       $value = self::tokenEscapeSmarty($value);
1811     }
1812     return $value;
1813   }
1814
1815   /**
1816    * @return array
1817    *   [legacy_token => new_token]
1818    */
1819   public static function legacyContactTokens() {
1820     return [
1821       'individual_prefix' => 'prefix_id',
1822       'individual_suffix' => 'suffix_id',
1823       'gender' => 'gender_id',
1824       'communication_style' => 'communication_style_id',
1825     ];
1826   }
1827
1828   /**
1829    * Get all custom field tokens of $entity
1830    *
1831    * @param string $entity
1832    * @param bool $usedForTokenWidget
1833    *
1834    * @return array
1835    *   return custom field tokens in array('custom_N' => 'label') format
1836    */
1837   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1838     $customTokens = [];
1839     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1840     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1841       $customTokens[sprintf($tokenName, $id)] = $info['label'] . ' :: ' . $info['groupTitle'];
1842     }
1843
1844     return $customTokens;
1845   }
1846
1847   /**
1848    * Formats a token list for the select2 widget
1849    *
1850    * @param $tokens
1851    * @return array
1852    */
1853   public static function formatTokensForDisplay($tokens) {
1854     $sorted = $output = [];
1855
1856     // Sort in ascending order by ignoring word case
1857     natcasesort($tokens);
1858
1859     // Attempt to place tokens into optgroups
1860     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1861     foreach ($tokens as $k => $v) {
1862       // Check to see if this token is already in a group e.g. for custom fields
1863       $split = explode(' :: ', $v);
1864       if (!empty($split[1])) {
1865         $sorted[$split[1]][] = ['id' => $k, 'text' => $split[0]];
1866       }
1867       // Group by entity
1868       else {
1869         $split = explode('.', trim($k, '{}'));
1870         if (isset($split[1])) {
1871           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucwords(str_replace('_', ' ', $split[0]));
1872         }
1873         else {
1874           $entity = 'Contact';
1875         }
1876         $sorted[ts($entity)][] = ['id' => $k, 'text' => $v];
1877       }
1878     }
1879
1880     ksort($sorted);
1881     foreach ($sorted as $k => $v) {
1882       $output[] = ['text' => $k, 'children' => $v];
1883     }
1884
1885     return $output;
1886   }
1887
1888   /**
1889    * @param $value
1890    * @param $token
1891    *
1892    * @return bool|int|mixed|string|null
1893    */
1894   protected static function convertPseudoConstantsUsingMetadata($value, $token) {
1895     // Convert pseudoconstants using metadata
1896     if ($value && is_numeric($value)) {
1897       $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
1898       if (!empty($allFields[$token]['pseudoconstant'])) {
1899         $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
1900       }
1901     }
1902     elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
1903       $value = CRM_Utils_Date::customFormat($value);
1904     }
1905     return $value;
1906   }
1907
1908   /**
1909    * Get token deprecation information.
1910    *
1911    * @return array
1912    */
1913   public static function getTokenDeprecations(): array {
1914     return [
1915       'WorkFlowMessageTemplates' => [
1916         'contribution_invoice_receipt' => [
1917           '$display_name' => 'contact.display_name',
1918         ],
1919         'contribution_online_receipt' => [
1920           '$contributeMode' => 'no longer available / relevant',
1921           '$first_name' => 'contact.first_name',
1922           '$last_name' => 'contact.last_name',
1923           '$displayName' => 'contact.display_name',
1924         ],
1925       ],
1926     ];
1927   }
1928
1929   /**
1930    * Get the tokens available for the domain.
1931    *
1932    * @param int $domainID
1933    * @param bool $html
1934    *
1935    * @return array
1936    * @throws \CRM_Core_Exception
1937    */
1938   protected static function getDomainTokens(int $domainID, bool $html): array {
1939     $cacheKey = __CLASS__ . 'domain_tokens' . $html . '_' . $domainID . '_' . CRM_Core_I18n::getLocale();
1940     if (!Civi::cache('metadata')->has($cacheKey)) {
1941       if (CRM_Core_Config::domainID() === $domainID) {
1942         $domain = CRM_Core_BAO_Domain::getDomain();
1943       }
1944       else {
1945         $domain = new CRM_Core_BAO_Domain();
1946         $domain->find(TRUE);
1947       }
1948       $tokens = [
1949         'name' => $domain->name,
1950         'id' => $domain->id,
1951         'description' => $domain->description,
1952       ];
1953       $loc = $domain->getLocationValues();
1954       if ($html) {
1955         $tokens['address'] = str_replace("\n", '<br />', ($loc['address'][1]['display'] ?? ''));
1956       }
1957       else {
1958         $tokens['address'] = $loc['address'][1]['display_text'] ?? '';
1959       }
1960       $phone = reset($loc['phone']);
1961       $email = reset($loc['email']);
1962       $tokens['phone'] = $phone['phone'] ?? '';
1963       $tokens['email'] = $email['email'] ?? '';
1964       Civi::cache('metadata')->set($cacheKey, $tokens);
1965     }
1966     return Civi::cache('metadata')->get($cacheKey);
1967   }
1968
1969 }