CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * Class to abstract token replacement.
  20  */
  21 class CRM_Utils_Token {
  22   public static $_requiredTokens = NULL;
  23
  24   public static $_tokens = [
  25     'action' => [
  26       'forward',
  27       'optOut',
  28       'optOutUrl',
  29       'reply',
  30       'unsubscribe',
  31       'unsubscribeUrl',
  32       'resubscribe',
  33       'resubscribeUrl',
  34       'subscribeUrl',
  35     ],
  36     'mailing' => [
  37       'id',
  38       'key',
  39       'name',
  40       'group',
  41       'subject',
  42       'viewUrl',
  43       'editUrl',
  44       'scheduleUrl',
  45       'approvalStatus',
  46       'approvalNote',
  47       'approveUrl',
  48       'creator',
  49       'creatorEmail',
  50     ],
  51     'user' => [
  52       // we extract the stuff after the role / permission and return the
  53       // civicrm email addresses of all users with that role / permission
  54       // useful with rules integration
  55       'permission:',
  56       'role:',
  57     ],
  58     // populate this dynamically
  59     'contact' => NULL,
  60     // populate this dynamically
  61     'contribution' => NULL,
  62     'domain' => [
  63       'name',
  64       'phone',
  65       'address',
  66       'email',
  67       'id',
  68       'description',
  69     ],
  70     'subscribe' => ['group'],
  71     'unsubscribe' => ['group'],
  72     'resubscribe' => ['group'],
  73     'welcome' => ['group'],
  74   ];
  75
  76   /**
  77    * @deprecated
  78    *   This is used by CiviMail but will be made redundant by FlexMailer.
  79    * @return array
  80    */
  81   public static function getRequiredTokens() {
  82     if (self::$_requiredTokens == NULL) {
  83       self::$_requiredTokens = [
  84         'domain.address' => ts("Domain address - displays your organization's postal address."),
  85         'action.optOutUrl or action.unsubscribeUrl' => [
  86           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
  87           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
  88           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
  89           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
  90         ],
  91       ];
  92     }
  93     return self::$_requiredTokens;
  94   }
  95
  96   /**
  97    * Check a string (mailing body) for required tokens.
  98    *
  99    * @param string $str
 100    *   The message.
 101    *
 102    * @return bool|array
 103    *   true if all required tokens are found,
 104    *    else an array of the missing tokens
 105    */
 106   public static function requiredTokens(&$str) {
 107     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 108     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, []) : CRM_Utils_Token::getRequiredTokens();
 109
 110     $missing = [];
 111     foreach ($requiredTokens as $token => $value) {
 112       if (!is_array($value)) {
 113         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 114           $missing[$token] = $value;
 115         }
 116       }
 117       else {
 118         $present = FALSE;
 119         $desc = NULL;
 120         foreach ($value as $t => $d) {
 121           $desc = $d;
 122           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 123             $present = TRUE;
 124           }
 125         }
 126         if (!$present) {
 127           $missing[$token] = $desc;
 128         }
 129       }
 130     }
 131
 132     if (empty($missing)) {
 133       return TRUE;
 134     }
 135     return $missing;
 136   }
 137
 138   /**
 139    * Wrapper for token matching.
 140    *
 141    * @param string $type
 142    *   The token type (domain,mailing,contact,action).
 143    * @param string $var
 144    *   The token variable.
 145    * @param string $str
 146    *   The string to search.
 147    *
 148    * @return bool
 149    *   Was there a match
 150    */
 151   public static function token_match($type, $var, &$str) {
 152     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 153     return preg_match("/(^|[^\{])$token/", $str);
 154   }
 155
 156   /**
 157    * Wrapper for token replacing.
 158    *
 159    * @param string $type
 160    *   The token type.
 161    * @param string $var
 162    *   The token variable.
 163    * @param string $value
 164    *   The value to substitute for the token.
 165    * @param string $str (reference) The string to replace in
 166    *
 167    * @param bool $escapeSmarty
 168    *
 169    * @return string
 170    *   The processed string
 171    */
 172   public static function token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 173     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 174     if (!$value) {
 175       $value = '$3';
 176     }
 177     if ($escapeSmarty) {
 178       $value = self::tokenEscapeSmarty($value);
 179     }
 180     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 181     return $str;
 182   }
 183
 184   /**
 185    * Get the regex for token replacement
 186    *
 187    * @param string $token_type
 188    *   A string indicating the the type of token to be used in the expression.
 189    *
 190    * @return string
 191    *   regular expression suitable for using in preg_replace
 192    */
 193   private static function tokenRegex(string $token_type) {
 194     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(:|\.)?\w*(\-[\w\s]+)?)\}(?!\})/';
 195   }
 196
 197   /**
 198    * Escape the string so a malicious user cannot inject smarty code into the template.
 199    *
 200    * @param string $string
 201    *   A string that needs to be escaped from smarty parsing.
 202    *
 203    * @return string
 204    *   the escaped string
 205    */
 206   public static function tokenEscapeSmarty($string) {
 207     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 208     return preg_replace(['/{/', '/(?<!{ldelim)}/'], ['{ldelim}', '{rdelim}'], $string);
 209   }
 210
 211   /**
 212    * Replace all the domain-level tokens in $str
 213    *
 214    * @param string $str
 215    *   The string with tokens to be replaced.
 216    * @param object $domain
 217    *   The domain BAO.
 218    * @param bool $html
 219    *   Replace tokens with HTML or plain text.
 220    *
 221    * @param null $knownTokens
 222    * @param bool $escapeSmarty
 223    *
 224    * @return string
 225    *   The processed string
 226    */
 227   public static function replaceDomainTokens(
 228     $str,
 229     $domain,
 230     $html = FALSE,
 231     $knownTokens = NULL,
 232     $escapeSmarty = FALSE
 233   ) {
 234     $key = 'domain';
 235     if (
 236       !$knownTokens || empty($knownTokens[$key])
 237     ) {
 238       return $str;
 239     }
 240
 241     $str = preg_replace_callback(
 242       self::tokenRegex($key),
 243       function ($matches) use ($domain, $html, $escapeSmarty) {
 244         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 245       },
 246       $str
 247     );
 248     return $str;
 249   }
 250
 251   /**
 252    * @param string $token
 253    * @param CRM_Core_BAO_Domain $domain
 254    * @param bool $html
 255    * @param bool $escapeSmarty
 256    *
 257    * @return null|string
 258    */
 259   public static function getDomainTokenReplacement($token, $domain, $html = FALSE, $escapeSmarty = FALSE): ?string {
 260     // check if the token we were passed is valid
 261     // we have to do this because this function is
 262     // called only when we find a token in the string
 263
 264     $loc = $domain->getLocationValues();
 265
 266     if (!in_array($token, self::$_tokens['domain'])) {
 267       $value = "{domain.$token}";
 268     }
 269     elseif ($token === 'address') {
 270       $cacheKey = __CLASS__ . 'address_token_cache' . CRM_Core_Config::domainID();
 271       $addressCache = Civi::cache()->has($cacheKey) ? Civi::cache()->get($cacheKey) : [];
 272
 273       $fieldKey = $html ? 'address-html' : 'address-text';
 274       if (array_key_exists($fieldKey, $addressCache)) {
 275         return $addressCache[$fieldKey];
 276       }
 277
 278       $value = NULL;
 279       // Construct the address token
 280
 281       if (!empty($loc[$token])) {
 282         if ($html) {
 283           $value = str_replace("\n", '<br />', $loc[$token][1]['display']);
 284         }
 285         else {
 286           $value = $loc[$token][1]['display_text'];
 287         }
 288         Civi::cache()->set($cacheKey, $addressCache);
 289       }
 290     }
 291     elseif ($token === 'name' || $token === 'id' || $token === 'description') {
 292       $value = $domain->$token;
 293     }
 294     elseif ($token === 'phone' || $token === 'email') {
 295       // Construct the phone and email tokens
 296
 297       $value = NULL;
 298       if (!empty($loc[$token])) {
 299         foreach ($loc[$token] as $index => $entity) {
 300           $value = $entity[$token];
 301           break;
 302         }
 303       }
 304     }
 305
 306     if ($escapeSmarty) {
 307       $value = self::tokenEscapeSmarty($value);
 308     }
 309
 310     return $value;
 311   }
 312
 313   /**
 314    * Replace all the org-level tokens in $str
 315    *
 316    * @fixme: This function appears to be broken, as it depended on
 317    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 318    * Marking as deprecated until this is clarified.
 319    *
 320    * @deprecated
 321    *  - the above hard-breakage was there from 2015 to 2021 and
 322    * no error was ever reported on it -does that mean
 323    * 1) the code is never hit because the only function that
 324    * calls this function is never called or
 325    * 2) it was called but never required to resolve any tokens
 326    * or more specifically custom field tokens
 327    *
 328    * The handling for custom fields with the removed token has
 329    * now been removed.
 330    *
 331    * @param string $str
 332    *   The string with tokens to be replaced.
 333    * @param object $org
 334    *   Associative array of org properties.
 335    * @param bool $html
 336    *   Replace tokens with HTML or plain text.
 337    *
 338    * @param bool $escapeSmarty
 339    *
 340    * @return string
 341    *   The processed string
 342    */
 343   public static function replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 344     self::$_tokens['org']
 345       = array_merge(
 346         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 347         ['address', 'display_name', 'checksum', 'contact_id']
 348       );
 349
 350     foreach (self::$_tokens['org'] as $token) {
 351       // print "Getting token value for $token<br/><br/>";
 352       if ($token === '') {
 353         continue;
 354       }
 355
 356       // If the string doesn't contain this token, skip it.
 357
 358       if (!self::token_match('org', $token, $str)) {
 359         continue;
 360       }
 361
 362       // Construct value from $token and $contact
 363
 364       $value = NULL;
 365
 366       if ($token === 'checksum') {
 367         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 368         $value = "cs={$cs}";
 369       }
 370       elseif ($token === 'address') {
 371         // Build the location values array
 372
 373         $loc = [];
 374         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 375         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 376         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 377         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 378         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 379
 380         // Construct the address token
 381
 382         $value = CRM_Utils_Address::format($loc);
 383         if ($html) {
 384           $value = str_replace("\n", '<br />', $value);
 385         }
 386       }
 387       else {
 388         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 389       }
 390
 391       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 392     }
 393
 394     return $str;
 395   }
 396
 397   /**
 398    * Replace all mailing tokens in $str
 399    *
 400    * @param string $str
 401    *   The string with tokens to be replaced.
 402    * @param object $mailing
 403    *   The mailing BAO, or null for validation.
 404    * @param bool $html
 405    *   Replace tokens with HTML or plain text.
 406    *
 407    * @param null $knownTokens
 408    * @param bool $escapeSmarty
 409    *
 410    * @return string
 411    *   The processed string
 412    */
 413   public static function &replaceMailingTokens(
 414     $str,
 415     &$mailing,
 416     $html = FALSE,
 417     $knownTokens = NULL,
 418     $escapeSmarty = FALSE
 419   ) {
 420     $key = 'mailing';
 421     if (!$knownTokens || !isset($knownTokens[$key])) {
 422       return $str;
 423     }
 424
 425     $str = preg_replace_callback(
 426       self::tokenRegex($key),
 427       function ($matches) use (&$mailing, $escapeSmarty) {
 428         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 429       },
 430       $str
 431     );
 432     return $str;
 433   }
 434
 435   /**
 436    * @param $token
 437    * @param $mailing
 438    * @param bool $escapeSmarty
 439    *
 440    * @return string
 441    */
 442   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 443     $value = '';
 444     switch ($token) {
 445       // CRM-7663
 446
 447       case 'id':
 448         $value = $mailing ? $mailing->id : 'undefined';
 449         break;
 450
 451       // Key is the ID, or the hash when the hash URLs setting is enabled
 452       case 'key':
 453         $value = $mailing->id;
 454         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($value)) {
 455           $value = $hash;
 456         }
 457         break;
 458
 459       case 'name':
 460         $value = $mailing ? $mailing->name : 'Mailing Name';
 461         break;
 462
 463       case 'group':
 464         $groups = $mailing ? $mailing->getGroupNames() : ['Mailing Groups'];
 465         $value = implode(', ', $groups);
 466         break;
 467
 468       case 'subject':
 469         $value = $mailing->subject;
 470         break;
 471
 472       case 'viewUrl':
 473         $mailingKey = $mailing->id;
 474         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 475           $mailingKey = $hash;
 476         }
 477         $value = CRM_Utils_System::url('civicrm/mailing/view',
 478           "reset=1&id={$mailingKey}",
 479           TRUE, NULL, FALSE, TRUE
 480         );
 481         break;
 482
 483       case 'editUrl':
 484       case 'scheduleUrl':
 485         // Note: editUrl and scheduleUrl used to be different, but now there's
 486         // one screen which can adapt based on permissions (in workflow mode).
 487         $value = CRM_Utils_System::url('civicrm/mailing/send',
 488           "reset=1&mid={$mailing->id}&continue=true",
 489           TRUE, NULL, FALSE, TRUE
 490         );
 491         break;
 492
 493       case 'html':
 494         $page = new CRM_Mailing_Page_View();
 495         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 496         break;
 497
 498       case 'approvalStatus':
 499         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 500         break;
 501
 502       case 'approvalNote':
 503         $value = $mailing->approval_note;
 504         break;
 505
 506       case 'approveUrl':
 507         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 508           "reset=1&mid={$mailing->id}",
 509           TRUE, NULL, FALSE, TRUE
 510         );
 511         break;
 512
 513       case 'creator':
 514         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 515         break;
 516
 517       case 'creatorEmail':
 518         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 519         break;
 520
 521       default:
 522         $value = "{mailing.$token}";
 523         break;
 524     }
 525
 526     if ($escapeSmarty) {
 527       $value = self::tokenEscapeSmarty($value);
 528     }
 529     return $value;
 530   }
 531
 532   /**
 533    * Replace all action tokens in $str
 534    *
 535    * @param string $str
 536    *   The string with tokens to be replaced.
 537    * @param array $addresses
 538    *   Assoc. array of VERP event addresses.
 539    * @param array $urls
 540    *   Assoc. array of action URLs.
 541    * @param bool $html
 542    *   Replace tokens with HTML or plain text.
 543    * @param array $knownTokens
 544    *   A list of tokens that are known to exist in the email body.
 545    *
 546    * @param bool $escapeSmarty
 547    *
 548    * @return string
 549    *   The processed string
 550    */
 551   public static function &replaceActionTokens(
 552     $str,
 553     &$addresses,
 554     &$urls,
 555     $html = FALSE,
 556     $knownTokens = NULL,
 557     $escapeSmarty = FALSE
 558   ) {
 559     $key = 'action';
 560     // here we intersect with the list of pre-configured valid tokens
 561     // so that we remove anything we do not recognize
 562     // I hope to move this step out of here soon and
 563     // then we will just iterate on a list of tokens that are passed to us
 564     if (!$knownTokens || empty($knownTokens[$key])) {
 565       return $str;
 566     }
 567
 568     $str = preg_replace_callback(
 569       self::tokenRegex($key),
 570       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 571         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 572       },
 573       $str
 574     );
 575     return $str;
 576   }
 577
 578   /**
 579    * @param $token
 580    * @param $addresses
 581    * @param $urls
 582    * @param bool $html
 583    * @param bool $escapeSmarty
 584    *
 585    * @return mixed|string
 586    */
 587   public static function getActionTokenReplacement(
 588     $token,
 589     &$addresses,
 590     &$urls,
 591     $html = FALSE,
 592     $escapeSmarty = FALSE
 593   ) {
 594     // If the token is an email action, use it.  Otherwise, find the
 595     // appropriate URL.
 596
 597     if (!in_array($token, self::$_tokens['action'])) {
 598       $value = "{action.$token}";
 599     }
 600     else {
 601       $value = $addresses[$token] ?? NULL;
 602
 603       if ($value == NULL) {
 604         $value = $urls[$token] ?? NULL;
 605       }
 606
 607       if ($value && $html) {
 608         // fix for CRM-2318
 609         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 610           $value = "mailto:$value";
 611         }
 612       }
 613       elseif ($value && !$html) {
 614         $value = str_replace('&amp;', '&', $value);
 615       }
 616     }
 617
 618     if ($escapeSmarty) {
 619       $value = self::tokenEscapeSmarty($value);
 620     }
 621     return $value;
 622   }
 623
 624   /**
 625    * Replace all the contact-level tokens in $str with information from
 626    * $contact.
 627    *
 628    * @param string $str
 629    *   The string with tokens to be replaced.
 630    * @param array $contact
 631    *   Associative array of contact properties.
 632    * @param bool $html
 633    *   Replace tokens with HTML or plain text.
 634    * @param array $knownTokens
 635    *   A list of tokens that are known to exist in the email body.
 636    * @param bool $returnBlankToken
 637    *   Return unevaluated token if value is null.
 638    *
 639    * @param bool $escapeSmarty
 640    *
 641    * @return string
 642    *   The processed string
 643    */
 644   public static function replaceContactTokens(
 645     $str,
 646     &$contact,
 647     $html = FALSE,
 648     $knownTokens = NULL,
 649     $returnBlankToken = FALSE,
 650     $escapeSmarty = FALSE
 651   ) {
 652     // Refresh contact tokens in case they have changed. There is heavy caching
 653     // in exportable fields so there is no benefit in doing this conditionally.
 654     self::$_tokens['contact'] = array_merge(
 655       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 656       ['checksum', 'contact_id']
 657     );
 658
 659     $key = 'contact';
 660     // here we intersect with the list of pre-configured valid tokens
 661     // so that we remove anything we do not recognize
 662     // I hope to move this step out of here soon and
 663     // then we will just iterate on a list of tokens that are passed to us
 664     if (!$knownTokens || empty($knownTokens[$key])) {
 665       return $str;
 666     }
 667
 668     $str = preg_replace_callback(
 669       self::tokenRegex($key),
 670       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 671         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 672       },
 673       $str
 674     );
 675
 676     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 677     return $str;
 678   }
 679
 680   /**
 681    * @param $token
 682    * @param $contact
 683    * @param bool $html
 684    * @param bool $returnBlankToken
 685    * @param bool $escapeSmarty
 686    *
 687    * @return bool|mixed|null|string
 688    */
 689   public static function getContactTokenReplacement(
 690     $token,
 691     &$contact,
 692     $html = FALSE,
 693     $returnBlankToken = FALSE,
 694     $escapeSmarty = FALSE
 695   ) {
 696     if (self::$_tokens['contact'] == NULL) {
 697       /* This should come from UF */
 698
 699       self::$_tokens['contact']
 700         = array_merge(
 701           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 702           ['checksum', 'contact_id']
 703         );
 704     }
 705
 706     // Construct value from $token and $contact
 707
 708     $value = NULL;
 709     $noReplace = FALSE;
 710
 711     // Support legacy tokens
 712     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 713
 714     // check if the token we were passed is valid
 715     // we have to do this because this function is
 716     // called only when we find a token in the string
 717
 718     if (!in_array($token, self::$_tokens['contact'])) {
 719       $noReplace = TRUE;
 720     }
 721     elseif ($token == 'checksum') {
 722       $hash = $contact['hash'] ?? NULL;
 723       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 724       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 725         NULL,
 726         NULL,
 727         $hash
 728       );
 729       $value = "cs={$cs}";
 730     }
 731     else {
 732       $value = (array) CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 733
 734       foreach ($value as $index => $item) {
 735         $value[$index] = self::convertPseudoConstantsUsingMetadata($value[$index], $token);
 736       }
 737       $value = implode(', ', $value);
 738     }
 739
 740     if (!$html) {
 741       $value = str_replace('&amp;', '&', $value);
 742     }
 743
 744     // if null then return actual token
 745     if ($returnBlankToken && !$value) {
 746       $noReplace = TRUE;
 747     }
 748
 749     if ($noReplace) {
 750       $value = "{contact.$token}";
 751     }
 752
 753     if ($escapeSmarty
 754       && !($returnBlankToken && $noReplace)
 755     ) {
 756       // $returnBlankToken means the caller wants to do further attempts at
 757       // processing unreplaced tokens -- so don't escape them yet in this case.
 758       $value = self::tokenEscapeSmarty($value);
 759     }
 760
 761     return $value;
 762   }
 763
 764   /**
 765    * Replace all the hook tokens in $str with information from
 766    * $contact.
 767    *
 768    * @param string $str
 769    *   The string with tokens to be replaced.
 770    * @param array $contact
 771    *   Associative array of contact properties (including hook token values).
 772    * @param $categories
 773    * @param bool $html
 774    *   Replace tokens with HTML or plain text.
 775    *
 776    * @param bool $escapeSmarty
 777    *
 778    * @return string
 779    *   The processed string
 780    */
 781   public static function &replaceHookTokens(
 782     $str,
 783     &$contact,
 784     $categories = NULL,
 785     $html = FALSE,
 786     $escapeSmarty = FALSE
 787   ) {
 788     if (!$categories) {
 789       $categories = self::getTokenCategories();
 790     }
 791     foreach ($categories as $key) {
 792       $str = preg_replace_callback(
 793         self::tokenRegex($key),
 794         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 795           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 796         },
 797         $str
 798       );
 799     }
 800     return $str;
 801   }
 802
 803   /**
 804    * Get the categories required for rendering tokens.
 805    *
 806    * @return array
 807    */
 808   public static function getTokenCategories(): array {
 809     if (!isset(\Civi::$statics[__CLASS__]['token_categories'])) {
 810       $tokens = [];
 811       \CRM_Utils_Hook::tokens($tokens);
 812       \Civi::$statics[__CLASS__]['token_categories'] = array_keys($tokens);
 813     }
 814     return \Civi::$statics[__CLASS__]['token_categories'];
 815   }
 816
 817   /**
 818    * Parse html through Smarty resolving any smarty functions.
 819    * @param string $tokenHtml
 820    * @param array $entity
 821    * @param string $entityType
 822    * @return string
 823    *   html parsed through smarty
 824    */
 825   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 826     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 827       $smarty = CRM_Core_Smarty::singleton();
 828       // also add the tokens to the template
 829       $smarty->assign_by_ref($entityType, $entity);
 830       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 831     }
 832     return $tokenHtml;
 833   }
 834
 835   /**
 836    * @param $token
 837    * @param $contact
 838    * @param $category
 839    * @param bool $html
 840    * @param bool $escapeSmarty
 841    *
 842    * @return mixed|string
 843    */
 844   public static function getHookTokenReplacement(
 845     $token,
 846     &$contact,
 847     $category,
 848     $html = FALSE,
 849     $escapeSmarty = FALSE
 850   ) {
 851     $value = $contact["{$category}.{$token}"] ?? NULL;
 852
 853     if ($value && !$html) {
 854       $value = str_replace('&amp;', '&', $value);
 855     }
 856
 857     if ($escapeSmarty) {
 858       $value = self::tokenEscapeSmarty($value);
 859     }
 860
 861     return $value;
 862   }
 863
 864   /**
 865    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 866    * for example {{token}}  or \{token}  will result in {token} in the final email
 867    *
 868    * this routine will remove the extra backslashes and braces
 869    *
 870    * @param $str ref to the string that will be scanned and modified
 871    */
 872   public static function unescapeTokens(&$str) {
 873     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 874   }
 875
 876   /**
 877    * Replace unsubscribe tokens.
 878    *
 879    * @param string $str
 880    *   The string with tokens to be replaced.
 881    * @param object $domain
 882    *   The domain BAO.
 883    * @param array $groups
 884    *   The groups (if any) being unsubscribed.
 885    * @param bool $html
 886    *   Replace tokens with html or plain text.
 887    * @param int $contact_id
 888    *   The contact ID.
 889    * @param string $hash The security hash of the unsub event
 890    *
 891    * @return string
 892    *   The processed string
 893    */
 894   public static function &replaceUnsubscribeTokens(
 895     $str,
 896     &$domain,
 897     &$groups,
 898     $html,
 899     $contact_id,
 900     $hash
 901   ) {
 902     if (self::token_match('unsubscribe', 'group', $str)) {
 903       if (!empty($groups)) {
 904         $config = CRM_Core_Config::singleton();
 905         $base = CRM_Utils_System::baseURL();
 906
 907         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 908         $dao = new CRM_Contact_DAO_Group();
 909         $dao->find();
 910         while ($dao->fetch()) {
 911           if (substr($dao->visibility, 0, 6) == 'Public') {
 912             $visibleGroups[] = $dao->id;
 913           }
 914         }
 915         $value = implode(', ', $groups);
 916         self::token_replace('unsubscribe', 'group', $value, $str);
 917       }
 918     }
 919     return $str;
 920   }
 921
 922   /**
 923    * Replace resubscribe tokens.
 924    *
 925    * @param string $str
 926    *   The string with tokens to be replaced.
 927    * @param object $domain
 928    *   The domain BAO.
 929    * @param array $groups
 930    *   The groups (if any) being resubscribed.
 931    * @param bool $html
 932    *   Replace tokens with html or plain text.
 933    * @param int $contact_id
 934    *   The contact ID.
 935    * @param string $hash The security hash of the resub event
 936    *
 937    * @return string
 938    *   The processed string
 939    */
 940   public static function &replaceResubscribeTokens(
 941     $str, &$domain, &$groups, $html,
 942     $contact_id, $hash
 943   ) {
 944     if (self::token_match('resubscribe', 'group', $str)) {
 945       if (!empty($groups)) {
 946         $value = implode(', ', $groups);
 947         self::token_replace('resubscribe', 'group', $value, $str);
 948       }
 949     }
 950     return $str;
 951   }
 952
 953   /**
 954    * Replace subscription-confirmation-request tokens
 955    *
 956    * @param string $str
 957    *   The string with tokens to be replaced.
 958    * @param string $group
 959    *   The name of the group being subscribed.
 960    * @param $url
 961    * @param bool $html
 962    *   Replace tokens with html or plain text.
 963    *
 964    * @return string
 965    *   The processed string
 966    */
 967   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 968     if (self::token_match('subscribe', 'group', $str)) {
 969       self::token_replace('subscribe', 'group', $group, $str);
 970     }
 971     if (self::token_match('subscribe', 'url', $str)) {
 972       self::token_replace('subscribe', 'url', $url, $str);
 973     }
 974     return $str;
 975   }
 976
 977   /**
 978    * Replace subscription-invitation tokens
 979    *
 980    * @param string $str
 981    *   The string with tokens to be replaced.
 982    *
 983    * @return string
 984    *   The processed string
 985    */
 986   public static function &replaceSubscribeInviteTokens($str) {
 987     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 988       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 989         'reset=1',
 990         TRUE, NULL, FALSE, TRUE
 991       );
 992       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 993     }
 994
 995     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 996       foreach ($matches as $key => $value) {
 997         $gid = substr($value, 21, -1);
 998         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 999           "reset=1&gid={$gid}",
1000           TRUE, NULL, FALSE, TRUE
1001         );
1002         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
1003       }
1004     }
1005
1006     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
1007       foreach ($matches as $key => $value) {
1008         $gid = substr($value, 18, -1);
1009         $config = CRM_Core_Config::singleton();
1010         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
1011         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
1012         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
1013         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1014       }
1015     }
1016     return $str;
1017   }
1018
1019   /**
1020    * Replace welcome/confirmation tokens
1021    *
1022    * @param string $str
1023    *   The string with tokens to be replaced.
1024    * @param string $group
1025    *   The name of the group being subscribed.
1026    * @param bool $html
1027    *   Replace tokens with html or plain text.
1028    *
1029    * @return string
1030    *   The processed string
1031    */
1032   public static function &replaceWelcomeTokens($str, $group, $html) {
1033     if (self::token_match('welcome', 'group', $str)) {
1034       self::token_replace('welcome', 'group', $group, $str);
1035     }
1036     return $str;
1037   }
1038
1039   /**
1040    * Find unprocessed tokens (call this last)
1041    *
1042    * @param string $str
1043    *   The string to search.
1044    *
1045    * @return array
1046    *   Array of tokens that weren't replaced
1047    */
1048   public static function &unmatchedTokens(&$str) {
1049     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1050     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1051     return $match[1];
1052   }
1053
1054   /**
1055    * Find and replace tokens for each component.
1056    *
1057    * @param string $str
1058    *   The string to search.
1059    * @param array $contact
1060    *   Associative array of contact properties.
1061    * @param array $components
1062    *   A list of tokens that are known to exist in the email body.
1063    *
1064    * @param bool $escapeSmarty
1065    * @param bool $returnEmptyToken
1066    *
1067    * @return string
1068    *   The processed string
1069    */
1070   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1071     if (!is_array($components) || empty($contact)) {
1072       return $str;
1073     }
1074
1075     foreach ($components as $name => $tokens) {
1076       if (!is_array($tokens) || empty($tokens)) {
1077         continue;
1078       }
1079
1080       foreach ($tokens as $token) {
1081         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1082           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1083         }
1084         elseif (!$returnEmptyToken) {
1085           //replacing empty token
1086           self::token_replace($name, $token, "", $str, $escapeSmarty);
1087         }
1088       }
1089     }
1090     return $str;
1091   }
1092
1093   /**
1094    * Get array of string tokens.
1095    *
1096    * @param string $string
1097    *   The input string to parse for tokens.
1098    *
1099    * @return array
1100    *   array of tokens mentioned in field
1101    */
1102   public static function getTokens($string) {
1103     $matches = [];
1104     $tokens = [];
1105     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+(:|.)?\w*)\}(?!\})/',
1106       $string,
1107       $matches,
1108       PREG_PATTERN_ORDER
1109     );
1110
1111     if ($matches[1]) {
1112       foreach ($matches[1] as $token) {
1113         $parts = explode('.', $token, 3);
1114         $type = $parts[0];
1115         $name = $parts[1];
1116         $suffix = !empty($parts[2]) ? ('.' . $parts[2]) : '';
1117         if ($name && $type) {
1118           if (!isset($tokens[$type])) {
1119             $tokens[$type] = [];
1120           }
1121           $tokens[$type][] = $name . $suffix;
1122         }
1123       }
1124     }
1125     return $tokens;
1126   }
1127
1128   /**
1129    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1130    * and getTokens appears to be historical rather than intentional and should be reviewed
1131    * @param $string
1132    * @return array
1133    *   fields to pass in as return properties when populating token
1134    */
1135   public static function getReturnProperties(&$string) {
1136     $returnProperties = [];
1137     $matches = [];
1138     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1139       $string,
1140       $matches,
1141       PREG_PATTERN_ORDER
1142     );
1143     if ($matches[1]) {
1144       foreach ($matches[1] as $token) {
1145         [$type, $name] = preg_split('/\./', $token, 2);
1146         if ($name) {
1147           $returnProperties["{$name}"] = 1;
1148         }
1149       }
1150     }
1151
1152     return $returnProperties;
1153   }
1154
1155   /**
1156    * Gives required details of contacts in an indexed array format so we
1157    * can iterate in a nice loop and do token evaluation
1158    *
1159    * @param array $contactIDs
1160    * @param array $returnProperties
1161    *   Of required properties.
1162    * @param bool $skipOnHold Don't return on_hold contact info also.
1163    *   Don't return on_hold contact info also.
1164    * @param bool $skipDeceased Don't return deceased contact info.
1165    *   Don't return deceased contact info.
1166    * @param array $extraParams
1167    *   Extra params - DEPRECATED
1168    * @param array $tokens
1169    *   The list of tokens we've extracted from the content.
1170    * @param string|null $className
1171    * @param int|null $jobID
1172    *   The mailing list jobID - this is a legacy param.
1173    *
1174    * @return array - e.g [[1 => ['first_name' => 'bob'...], 34 => ['first_name' => 'fred'...]]]
1175    */
1176   public static function getTokenDetails(
1177     $contactIDs,
1178     $returnProperties = NULL,
1179     $skipOnHold = TRUE,
1180     $skipDeceased = TRUE,
1181     $extraParams = NULL,
1182     $tokens = [],
1183     $className = NULL,
1184     $jobID = NULL
1185   ) {
1186
1187     $params = [];
1188     foreach ($contactIDs as $contactID) {
1189       $params[] = [
1190         CRM_Core_Form::CB_PREFIX . $contactID,
1191         '=',
1192         1,
1193         0,
1194         0,
1195       ];
1196     }
1197
1198     // fix for CRM-2613
1199     if ($skipDeceased) {
1200       $params[] = ['is_deceased', '=', 0, 0, 0];
1201     }
1202
1203     //fix for CRM-3798
1204     if ($skipOnHold) {
1205       $params[] = ['on_hold', '=', 0, 0, 0];
1206     }
1207
1208     if ($extraParams) {
1209       CRM_Core_Error::deprecatedWarning('Passing $extraParams to getTokenDetails() is not supported and will be removed in a future version');
1210       $params = array_merge($params, $extraParams);
1211     }
1212
1213     // if return properties are not passed then get all return properties
1214     if (empty($returnProperties)) {
1215       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1216         ['display_name', 'checksum', 'contact_id']
1217       );
1218       foreach ($fields as $val) {
1219         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1220         // with contacts and are expensive to resolve.
1221         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1222         // and upstream calls to this should populate return properties.
1223         $unavailableFields = ['group', 'tag'];
1224         if (!in_array($val, $unavailableFields)) {
1225           $returnProperties[$val] = 1;
1226         }
1227       }
1228     }
1229
1230     $custom = [];
1231     foreach ($returnProperties as $name => $dontCare) {
1232       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1233       if ($cfID) {
1234         $custom[] = $cfID;
1235       }
1236     }
1237
1238     [$contactDetails] = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1239
1240     foreach ($contactIDs as $contactID) {
1241       if (array_key_exists($contactID, $contactDetails)) {
1242         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1243         ) {
1244           $communicationPreferences = [];
1245           foreach ((array) $contactDetails[$contactID]['preferred_communication_method'] as $val) {
1246             if ($val) {
1247               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1248             }
1249           }
1250           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1251         }
1252
1253         foreach ($custom as $cfID) {
1254           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1255             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1256           }
1257         }
1258
1259         // special case for greeting replacement
1260         foreach ([
1261           'email_greeting',
1262           'postal_greeting',
1263           'addressee',
1264         ] as $val) {
1265           if (!empty($contactDetails[$contactID][$val])) {
1266             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1267           }
1268         }
1269       }
1270     }
1271
1272     // $contactDetails = &$details[0] = is an array of [ contactID => contactDetails ]
1273     // also call a hook and get token details
1274     CRM_Utils_Hook::tokenValues($contactDetails,
1275       $contactIDs,
1276       $jobID,
1277       $tokens,
1278       $className
1279     );
1280     return [$contactDetails];
1281   }
1282
1283   /**
1284    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1285    * specific tokens to be rendered
1286    *
1287    * @param array $contactIDs
1288    *   This should always be array(0) or its not anonymous - left to keep signature same.
1289    *   as main fn
1290    * @param string $returnProperties
1291    * @param bool $skipOnHold
1292    * @param bool $skipDeceased
1293    * @param string $extraParams
1294    * @param array $tokens
1295    * @param string $className
1296    *   Sent as context to the hook.
1297    * @param string $jobID
1298    * @return array
1299    *   contactDetails with hooks swapped out
1300    */
1301   public static function getAnonymousTokenDetails($contactIDs = [0],
1302                                            $returnProperties = NULL,
1303                                            $skipOnHold = TRUE,
1304                                            $skipDeceased = TRUE,
1305                                            $extraParams = NULL,
1306                                            $tokens = [],
1307                                            $className = NULL,
1308                                            $jobID = NULL) {
1309     $details = [0 => []];
1310     // also call a hook and get token details
1311     CRM_Utils_Hook::tokenValues($details[0],
1312       $contactIDs,
1313       $jobID,
1314       $tokens,
1315       $className
1316     );
1317     return $details;
1318   }
1319
1320   /**
1321    * Get Membership Token Details.
1322    * @param array $membershipIDs
1323    *   Array of membership IDS.
1324    */
1325   public static function getMembershipTokenDetails($membershipIDs) {
1326     $memberships = civicrm_api3('membership', 'get', [
1327       'options' => ['limit' => 0],
1328       'membership_id' => ['IN' => (array) $membershipIDs],
1329     ]);
1330     return $memberships['values'];
1331   }
1332
1333   /**
1334    * Replace existing greeting tokens in message/subject.
1335    *
1336    * This function operates by reference, modifying the first parameter. Other
1337    * methods for token replacement in this class return the modified string.
1338    * This leads to inconsistency in how these methods must be applied.
1339    *
1340    * @TODO Remove that inconsistency in usage.
1341    *
1342    * ::replaceContactTokens() may need to be called after this method, to
1343    * replace tokens supplied from this method.
1344    *
1345    * @param string $tokenString
1346    * @param array $contactDetails
1347    * @param int $contactId
1348    * @param string $className
1349    * @param bool $escapeSmarty
1350    */
1351   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1352
1353     if (!$contactDetails && !$contactId) {
1354       return;
1355     }
1356
1357     // check if there are any tokens
1358     $greetingTokens = self::getTokens($tokenString);
1359
1360     if (!empty($greetingTokens)) {
1361       // first use the existing contact object for token replacement
1362       if (!empty($contactDetails)) {
1363         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1364       }
1365
1366       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1367       // check if there are any unevaluated tokens
1368       $greetingTokens = self::getTokens($tokenString);
1369
1370       // $greetingTokens not empty, means there are few tokens which are not
1371       // evaluated, like custom data etc
1372       // so retrieve it from database
1373       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1374         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1375         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1376         $contactParams = ['contact_id' => $contactId];
1377
1378         $greetingDetails = self::getTokenDetails($contactParams,
1379           $greetingsReturnProperties,
1380           FALSE, FALSE, NULL,
1381           $greetingTokens,
1382           $className
1383         );
1384
1385         // again replace tokens
1386         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1387           $greetingDetails,
1388           TRUE,
1389           $greetingTokens,
1390           TRUE,
1391           $escapeSmarty
1392         );
1393       }
1394
1395       // check if there are still any unevaluated tokens
1396       $remainingTokens = self::getTokens($tokenString);
1397
1398       // $greetingTokens not empty, there are customized or hook tokens to replace
1399       if (!empty($remainingTokens)) {
1400         // Fill the return properties array
1401         $greetingTokens = $remainingTokens;
1402         reset($greetingTokens);
1403         $greetingsReturnProperties = [];
1404         foreach ($greetingTokens as $value) {
1405           $props = array_flip($value);
1406           $props = array_fill_keys(array_keys($props), 1);
1407           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1408         }
1409         $contactParams = ['contact_id' => $contactId];
1410         $greetingDetails = self::getTokenDetails($contactParams,
1411           $greetingsReturnProperties,
1412           FALSE, FALSE, NULL,
1413           $greetingTokens,
1414           $className
1415         );
1416         // Prepare variables for calling replaceHookTokens
1417         $categories = array_keys($greetingTokens);
1418         [$contact] = $greetingDetails;
1419         // Replace tokens defined in Hooks.
1420         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1421       }
1422     }
1423   }
1424
1425   /**
1426    * At this point, $contactDetails has loaded the contact from the DAO. Any
1427    * (non-custom) missing fields are null.  By removing them, we can avoid
1428    * expensive calls to CRM_Contact_BAO_Query.
1429    *
1430    * @param string $tokenString
1431    * @param array $contactDetails
1432    * @param array $greetingTokens
1433    */
1434   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1435
1436     // Only applies to contact tokens
1437     if (!array_key_exists('contact', $greetingTokens)) {
1438       return;
1439     }
1440
1441     $greetingTokensOriginal = $greetingTokens;
1442     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1443     // Sometimes contactDetails are in a multidemensional array, sometimes a
1444     // single-dimension array.
1445     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1446       $contactDetails = current($contactDetails[0]);
1447     }
1448     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1449
1450     // Handle legacy tokens
1451     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1452       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1453         $nullFields[] = $oldToken;
1454       }
1455     }
1456
1457     // Remove null contact fields from $greetingTokens
1458     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1459
1460     // Also remove them from $tokenString
1461     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1462     // Handle legacy tokens again, sigh
1463     if (!empty($removedTokens)) {
1464       foreach ($removedTokens as $token) {
1465         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1466           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1467         }
1468       }
1469       foreach ($removedTokens as $token) {
1470         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1471       }
1472     }
1473   }
1474
1475   /**
1476    * @param $tokens
1477    *
1478    * @return array
1479    */
1480   public static function flattenTokens(&$tokens) {
1481     $flattenTokens = [];
1482
1483     foreach ([
1484       'html',
1485       'text',
1486       'subject',
1487     ] as $prop) {
1488       if (!isset($tokens[$prop])) {
1489         continue;
1490       }
1491       foreach ($tokens[$prop] as $type => $names) {
1492         if (!isset($flattenTokens[$type])) {
1493           $flattenTokens[$type] = [];
1494         }
1495         foreach ($names as $name) {
1496           $flattenTokens[$type][$name] = 1;
1497         }
1498       }
1499     }
1500
1501     return $flattenTokens;
1502   }
1503
1504   /**
1505    * Replace all user tokens in $str
1506    *
1507    * @param string $str
1508    *   The string with tokens to be replaced.
1509    *
1510    * @param null $knownTokens
1511    * @param bool $escapeSmarty
1512    *
1513    * @return string
1514    *   The processed string
1515    */
1516   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1517     $key = 'user';
1518     if (!$knownTokens ||
1519       !isset($knownTokens[$key])
1520     ) {
1521       return $str;
1522     }
1523
1524     $str = preg_replace_callback(
1525       self::tokenRegex($key),
1526       function ($matches) use ($escapeSmarty) {
1527         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1528       },
1529       $str
1530     );
1531     return $str;
1532   }
1533
1534   /**
1535    * @param $token
1536    * @param bool $escapeSmarty
1537    *
1538    * @return string
1539    */
1540   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1541     $value = '';
1542
1543     [$objectName, $objectValue] = explode('-', $token, 2);
1544
1545     switch ($objectName) {
1546       case 'permission':
1547         $value = CRM_Core_Permission::permissionEmails($objectValue);
1548         break;
1549
1550       case 'role':
1551         $value = CRM_Core_Permission::roleEmails($objectValue);
1552         break;
1553     }
1554
1555     if ($escapeSmarty) {
1556       $value = self::tokenEscapeSmarty($value);
1557     }
1558
1559     return $value;
1560   }
1561
1562   protected static function _buildContributionTokens() {
1563     $key = 'contribution';
1564
1565     if (!isset(Civi::$statics[__CLASS__][__FUNCTION__][$key])) {
1566       $processor = new CRM_Contribute_Tokens();
1567       $tokens = array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1568         ['campaign' => [], 'financial_type' => [], 'payment_instrument' => []],
1569         self::getCustomFieldTokens('Contribution'),
1570         $processor->getPseudoTokens()
1571       );
1572       foreach ($tokens as $token) {
1573         if (!empty($token['name'])) {
1574           $tokens[$token['name']] = [];
1575         }
1576       }
1577       Civi::$statics[__CLASS__][__FUNCTION__][$key] = array_keys($tokens);
1578     }
1579     self::$_tokens[$key] = Civi::$statics[__CLASS__][__FUNCTION__][$key];
1580   }
1581
1582   /**
1583    * Store membership tokens on the static _tokens array.
1584    */
1585   protected static function _buildMembershipTokens() {
1586     $key = 'membership';
1587     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1588       $membershipTokens = [];
1589       $tokens = CRM_Core_SelectValues::membershipTokens();
1590       foreach ($tokens as $token => $dontCare) {
1591         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1592       }
1593       self::$_tokens[$key] = $membershipTokens;
1594     }
1595   }
1596
1597   /**
1598    * Replace tokens for an entity.
1599    * @param string $entity
1600    * @param array $entityArray
1601    *   (e.g. in format from api).
1602    * @param string $str
1603    *   String to replace in.
1604    * @param array $knownTokens
1605    *   Array of tokens present.
1606    * @param bool $escapeSmarty
1607    * @return string
1608    *   string with replacements made
1609    */
1610   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1611     if (!$knownTokens || empty($knownTokens[$entity])) {
1612       return $str;
1613     }
1614
1615     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1616     $fn = is_callable(['CRM_Utils_Token', $fn]) ? $fn : 'getApiTokenReplacement';
1617     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1618     foreach ($knownTokens[$entity] as $token) {
1619       $replacement = self::$fn($entity, $token, $entityArray);
1620       if ($escapeSmarty) {
1621         $replacement = self::tokenEscapeSmarty($replacement);
1622       }
1623       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1624     }
1625     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1626   }
1627
1628   /**
1629    * @param int $caseId
1630    * @param string $str
1631    * @param array $knownTokens
1632    * @param bool $escapeSmarty
1633    * @return string
1634    * @throws \CiviCRM_API3_Exception
1635    */
1636   public static function replaceCaseTokens($caseId, $str, $knownTokens = NULL, $escapeSmarty = FALSE): string {
1637     if (strpos($str, '{case.') === FALSE) {
1638       return $str;
1639     }
1640     if (!$knownTokens) {
1641       $knownTokens = self::getTokens($str);
1642     }
1643     $case = civicrm_api3('case', 'getsingle', ['id' => $caseId]);
1644     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1645   }
1646
1647   /**
1648    * Generic function for formatting token replacement for an api field
1649    *
1650    * @param string $entity
1651    * @param string $token
1652    * @param array $entityArray
1653    * @return string
1654    * @throws \CiviCRM_API3_Exception
1655    */
1656   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1657     if (!isset($entityArray[$token])) {
1658       return '';
1659     }
1660     $field = civicrm_api3($entity, 'getfield', ['action' => 'get', 'name' => $token, 'get_options' => 'get']);
1661     $field = $field['values'];
1662     $fieldType = $field['type'] ?? NULL;
1663     // Boolean fields
1664     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1665       $field['options'] = [ts('No'), ts('Yes')];
1666     }
1667     // Match pseudoconstants
1668     if (!empty($field['options'])) {
1669       $ret = [];
1670       foreach ((array) $entityArray[$token] as $val) {
1671         $ret[] = $field['options'][$val];
1672       }
1673       return implode(', ', $ret);
1674     }
1675     // Format date fields
1676     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1677       return CRM_Utils_Date::customFormat($entityArray[$token]);
1678     }
1679     return implode(', ', (array) $entityArray[$token]);
1680   }
1681
1682   /**
1683    * Replace Contribution tokens in html.
1684    *
1685    * @param string $str
1686    * @param array $contribution
1687    * @param bool|string $html
1688    * @param string $knownTokens
1689    * @param bool|string $escapeSmarty
1690    *
1691    * @return mixed
1692    */
1693   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1694     $key = 'contribution';
1695     if (!$knownTokens || empty($knownTokens[$key])) {
1696       //early return
1697       return $str;
1698     }
1699
1700     // here we intersect with the list of pre-configured valid tokens
1701     // so that we remove anything we do not recognize
1702     // I hope to move this step out of here soon and
1703     // then we will just iterate on a list of tokens that are passed to us
1704
1705     $str = preg_replace_callback(
1706       self::tokenRegex($key),
1707       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1708         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1709       },
1710       $str
1711     );
1712
1713     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1714     return $str;
1715   }
1716
1717   /**
1718    * We have a situation where we are rendering more than one token in each field because we are combining
1719    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1720    *
1721    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1722    * it applies (other tokens are OK including date fields)
1723    *
1724    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1725    * does - but the fields is already being formatted through a date function
1726    *
1727    * @param string $separator
1728    * @param string $str
1729    * @param array $contributions
1730    * @param array $knownTokens
1731    *
1732    * @return string
1733    */
1734   public static function replaceMultipleContributionTokens(string $separator, string $str, array $contributions, array $knownTokens): string {
1735     foreach ($knownTokens['contribution'] ?? [] as $token) {
1736       $resolvedTokens = [];
1737       foreach ($contributions as $contribution) {
1738         $resolvedTokens[] = self::replaceContributionTokens('{contribution.' . $token . '}', $contribution, FALSE, $knownTokens);
1739       }
1740       $str = self::token_replace('contribution', $token, implode($separator, $resolvedTokens), $str);
1741     }
1742     return $str;
1743   }
1744
1745   /**
1746    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1747    * - this is used by the pdfLetter task from membership search
1748    * @param string $entity
1749    *   should always be "membership"
1750    * @param string $token
1751    *   field name
1752    * @param array $membership
1753    *   An api result array for a single membership.
1754    * @return string token replacement
1755    */
1756   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1757     self::_buildMembershipTokens();
1758     switch ($token) {
1759       case 'type':
1760         $value = $membership['membership_name'];
1761         break;
1762
1763       case 'status':
1764         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1765         $value = $statuses[$membership['status_id']];
1766         break;
1767
1768       case 'fee':
1769         try {
1770           $value = civicrm_api3('membership_type', 'getvalue', [
1771             'id' => $membership['membership_type_id'],
1772             'return' => 'minimum_fee',
1773           ]);
1774           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1775         }
1776         catch (CiviCRM_API3_Exception $e) {
1777           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1778           // api handles NULL (4.4)
1779           $value = 0;
1780         }
1781         break;
1782
1783       default:
1784         if (in_array($token, self::$_tokens[$entity])) {
1785           $value = $membership[$token];
1786           if (CRM_Utils_String::endsWith($token, '_date')) {
1787             $value = CRM_Utils_Date::customFormat($value);
1788           }
1789         }
1790         else {
1791           // ie unchanged
1792           $value = "{$entity}.{$token}";
1793         }
1794         break;
1795     }
1796
1797     return $value;
1798   }
1799
1800   /**
1801    * @param $token
1802    * @param $contribution
1803    * @param bool $html
1804    * @param bool $escapeSmarty
1805    *
1806    * @return mixed|string
1807    * @throws \CRM_Core_Exception
1808    */
1809   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1810     self::_buildContributionTokens();
1811
1812     switch ($token) {
1813       case 'total_amount':
1814       case 'net_amount':
1815       case 'fee_amount':
1816       case 'non_deductible_amount':
1817         // FIXME: Is this ever a multi-dimensional array?  Why use retrieveValueRecursive()?
1818         $amount = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1819         $currency = CRM_Utils_Array::retrieveValueRecursive($contribution, 'currency');
1820         $value = CRM_Utils_Money::format($amount, $currency);
1821         break;
1822
1823       case 'receive_date':
1824       case 'receipt_date':
1825         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1826         $config = CRM_Core_Config::singleton();
1827         $value = CRM_Utils_Date::customFormat($value, $config->dateformatDatetime);
1828         break;
1829
1830       default:
1831         if (!in_array($token, self::$_tokens['contribution'])) {
1832           $value = "{contribution.$token}";
1833         }
1834         else {
1835           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1836         }
1837         break;
1838     }
1839
1840     if ($escapeSmarty) {
1841       $value = self::tokenEscapeSmarty($value);
1842     }
1843     return $value;
1844   }
1845
1846   /**
1847    * @return array
1848    *   [legacy_token => new_token]
1849    */
1850   public static function legacyContactTokens() {
1851     return [
1852       'individual_prefix' => 'prefix_id',
1853       'individual_suffix' => 'suffix_id',
1854       'gender' => 'gender_id',
1855       'communication_style' => 'communication_style_id',
1856     ];
1857   }
1858
1859   /**
1860    * Get all custom field tokens of $entity
1861    *
1862    * @param string $entity
1863    * @param bool $usedForTokenWidget
1864    *
1865    * @return array
1866    *   return custom field tokens in array('custom_N' => 'label') format
1867    */
1868   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1869     $customTokens = [];
1870     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1871     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1872       $customTokens[sprintf($tokenName, $id)] = $info['label'];
1873     }
1874
1875     return $customTokens;
1876   }
1877
1878   /**
1879    * Formats a token list for the select2 widget
1880    *
1881    * @param $tokens
1882    * @return array
1883    */
1884   public static function formatTokensForDisplay($tokens) {
1885     $sorted = $output = [];
1886
1887     // Sort in ascending order by ignoring word case
1888     natcasesort($tokens);
1889
1890     // Attempt to place tokens into optgroups
1891     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1892     foreach ($tokens as $k => $v) {
1893       // Check to see if this token is already in a group e.g. for custom fields
1894       $split = explode(' :: ', $v);
1895       if (!empty($split[1])) {
1896         $sorted[$split[1]][] = ['id' => $k, 'text' => $split[0]];
1897       }
1898       // Group by entity
1899       else {
1900         $split = explode('.', trim($k, '{}'));
1901         if (isset($split[1])) {
1902           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucwords(str_replace('_', ' ', $split[0]));
1903         }
1904         else {
1905           $entity = 'Contact';
1906         }
1907         $sorted[ts($entity)][] = ['id' => $k, 'text' => $v];
1908       }
1909     }
1910
1911     ksort($sorted);
1912     foreach ($sorted as $k => $v) {
1913       $output[] = ['text' => $k, 'children' => $v];
1914     }
1915
1916     return $output;
1917   }
1918
1919   /**
1920    * @param $value
1921    * @param $token
1922    *
1923    * @return bool|int|mixed|string|null
1924    */
1925   protected static function convertPseudoConstantsUsingMetadata($value, $token) {
1926     // Convert pseudoconstants using metadata
1927     if ($value && is_numeric($value)) {
1928       $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
1929       if (!empty($allFields[$token]['pseudoconstant'])) {
1930         $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
1931       }
1932     }
1933     elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
1934       $value = CRM_Utils_Date::customFormat($value);
1935     }
1936     return $value;
1937   }
1938
1939   /**
1940    * Get token deprecation information.
1941    *
1942    * @return array
1943    */
1944   public static function getTokenDeprecations(): array {
1945     return [
1946       'WorkFlowMessageTemplates' => [
1947         'contribution_invoice_receipt' => [
1948           '$display_name' => 'contact.display_name',
1949         ],
1950       ],
1951     ];
1952   }
1953
1954 }