CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2020                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2020
  32  */
  33
  34 /**
  35  * Class to abstract token replacement.
  36  */
  37 class CRM_Utils_Token {
  38   public static $_requiredTokens = NULL;
  39
  40   public static $_tokens = [
  41     'action' => [
  42       'forward',
  43       'optOut',
  44       'optOutUrl',
  45       'reply',
  46       'unsubscribe',
  47       'unsubscribeUrl',
  48       'resubscribe',
  49       'resubscribeUrl',
  50       'subscribeUrl',
  51     ],
  52     'mailing' => [
  53       'id',
  54       'name',
  55       'group',
  56       'subject',
  57       'viewUrl',
  58       'editUrl',
  59       'scheduleUrl',
  60       'approvalStatus',
  61       'approvalNote',
  62       'approveUrl',
  63       'creator',
  64       'creatorEmail',
  65     ],
  66     'user' => [
  67       // we extract the stuff after the role / permission and return the
  68       // civicrm email addresses of all users with that role / permission
  69       // useful with rules integration
  70       'permission:',
  71       'role:',
  72     ],
  73     // populate this dynamically
  74     'contact' => NULL,
  75     // populate this dynamically
  76     'contribution' => NULL,
  77     'domain' => [
  78       'name',
  79       'phone',
  80       'address',
  81       'email',
  82       'id',
  83       'description',
  84     ],
  85     'subscribe' => ['group'],
  86     'unsubscribe' => ['group'],
  87     'resubscribe' => ['group'],
  88     'welcome' => ['group'],
  89   ];
  90
  91   /**
  92    * @deprecated
  93    *   This is used by CiviMail but will be made redundant by FlexMailer.
  94    * @return array
  95    */
  96   public static function getRequiredTokens() {
  97     if (self::$_requiredTokens == NULL) {
  98       self::$_requiredTokens = [
  99         'domain.address' => ts("Domain address - displays your organization's postal address."),
 100         'action.optOutUrl or action.unsubscribeUrl' => [
 101           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
 102           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
 103           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
 104           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
 105         ],
 106       ];
 107     }
 108     return self::$_requiredTokens;
 109   }
 110
 111   /**
 112    * Check a string (mailing body) for required tokens.
 113    *
 114    * @param string $str
 115    *   The message.
 116    *
 117    * @return bool|array
 118    *   true if all required tokens are found,
 119    *    else an array of the missing tokens
 120    */
 121   public static function requiredTokens(&$str) {
 122     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 123     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, []) : CRM_Utils_Token::getRequiredTokens();
 124
 125     $missing = [];
 126     foreach ($requiredTokens as $token => $value) {
 127       if (!is_array($value)) {
 128         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 129           $missing[$token] = $value;
 130         }
 131       }
 132       else {
 133         $present = FALSE;
 134         $desc = NULL;
 135         foreach ($value as $t => $d) {
 136           $desc = $d;
 137           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 138             $present = TRUE;
 139           }
 140         }
 141         if (!$present) {
 142           $missing[$token] = $desc;
 143         }
 144       }
 145     }
 146
 147     if (empty($missing)) {
 148       return TRUE;
 149     }
 150     return $missing;
 151   }
 152
 153   /**
 154    * Wrapper for token matching.
 155    *
 156    * @param string $type
 157    *   The token type (domain,mailing,contact,action).
 158    * @param string $var
 159    *   The token variable.
 160    * @param string $str
 161    *   The string to search.
 162    *
 163    * @return bool
 164    *   Was there a match
 165    */
 166   public static function token_match($type, $var, &$str) {
 167     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 168     return preg_match("/(^|[^\{])$token/", $str);
 169   }
 170
 171   /**
 172    * Wrapper for token replacing.
 173    *
 174    * @param string $type
 175    *   The token type.
 176    * @param string $var
 177    *   The token variable.
 178    * @param string $value
 179    *   The value to substitute for the token.
 180    * @param string $str (reference) The string to replace in
 181    *
 182    * @param bool $escapeSmarty
 183    *
 184    * @return string
 185    *   The processed string
 186    */
 187   public static function &token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 188     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 189     if (!$value) {
 190       $value = '$3';
 191     }
 192     if ($escapeSmarty) {
 193       $value = self::tokenEscapeSmarty($value);
 194     }
 195     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 196     return $str;
 197   }
 198
 199   /**
 200    * Get< the regex for token replacement
 201    *
 202    * @param string $token_type
 203    *   A string indicating the the type of token to be used in the expression.
 204    *
 205    * @return string
 206    *   regular expression sutiable for using in preg_replace
 207    */
 208   private static function tokenRegex($token_type) {
 209     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(\-[\w\s]+)?)\}(?!\})/';
 210   }
 211
 212   /**
 213    * Escape the string so a malicious user cannot inject smarty code into the template.
 214    *
 215    * @param string $string
 216    *   A string that needs to be escaped from smarty parsing.
 217    *
 218    * @return string
 219    *   the escaped string
 220    */
 221   public static function tokenEscapeSmarty($string) {
 222     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 223     return preg_replace(['/{/', '/(?<!{ldelim)}/'], ['{ldelim}', '{rdelim}'], $string);
 224   }
 225
 226   /**
 227    * Replace all the domain-level tokens in $str
 228    *
 229    * @param string $str
 230    *   The string with tokens to be replaced.
 231    * @param object $domain
 232    *   The domain BAO.
 233    * @param bool $html
 234    *   Replace tokens with HTML or plain text.
 235    *
 236    * @param null $knownTokens
 237    * @param bool $escapeSmarty
 238    *
 239    * @return string
 240    *   The processed string
 241    */
 242   public static function &replaceDomainTokens(
 243     $str,
 244     &$domain,
 245     $html = FALSE,
 246     $knownTokens = NULL,
 247     $escapeSmarty = FALSE
 248   ) {
 249     $key = 'domain';
 250     if (
 251       !$knownTokens || empty($knownTokens[$key])
 252     ) {
 253       return $str;
 254     }
 255
 256     $str = preg_replace_callback(
 257       self::tokenRegex($key),
 258       function ($matches) use (&$domain, $html, $escapeSmarty) {
 259         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 260       },
 261       $str
 262     );
 263     return $str;
 264   }
 265
 266   /**
 267    * @param $token
 268    * @param $domain
 269    * @param bool $html
 270    * @param bool $escapeSmarty
 271    *
 272    * @return mixed|null|string
 273    */
 274   public static function getDomainTokenReplacement($token, &$domain, $html = FALSE, $escapeSmarty = FALSE) {
 275     // check if the token we were passed is valid
 276     // we have to do this because this function is
 277     // called only when we find a token in the string
 278
 279     $loc = &$domain->getLocationValues();
 280
 281     if (!in_array($token, self::$_tokens['domain'])) {
 282       $value = "{domain.$token}";
 283     }
 284     elseif ($token == 'address') {
 285       static $addressCache = [];
 286
 287       $cache_key = $html ? 'address-html' : 'address-text';
 288       if (array_key_exists($cache_key, $addressCache)) {
 289         return $addressCache[$cache_key];
 290       }
 291
 292       $value = NULL;
 293       // Construct the address token
 294
 295       if (!empty($loc[$token])) {
 296         if ($html) {
 297           $value = $loc[$token][1]['display'];
 298           $value = str_replace("\n", '<br />', $value);
 299         }
 300         else {
 301           $value = $loc[$token][1]['display_text'];
 302         }
 303         $addressCache[$cache_key] = $value;
 304       }
 305     }
 306     elseif ($token == 'name' || $token == 'id' || $token == 'description') {
 307       $value = $domain->$token;
 308     }
 309     elseif ($token == 'phone' || $token == 'email') {
 310       // Construct the phone and email tokens
 311
 312       $value = NULL;
 313       if (!empty($loc[$token])) {
 314         foreach ($loc[$token] as $index => $entity) {
 315           $value = $entity[$token];
 316           break;
 317         }
 318       }
 319     }
 320
 321     if ($escapeSmarty) {
 322       $value = self::tokenEscapeSmarty($value);
 323     }
 324
 325     return $value;
 326   }
 327
 328   /**
 329    * Replace all the org-level tokens in $str
 330    *
 331    * @fixme: This function appears to be broken, as it depends on
 332    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 333    * Marking as deprecated until this is fixed
 334    * @deprecated
 335    *
 336    * @param string $str
 337    *   The string with tokens to be replaced.
 338    * @param object $org
 339    *   Associative array of org properties.
 340    * @param bool $html
 341    *   Replace tokens with HTML or plain text.
 342    *
 343    * @param bool $escapeSmarty
 344    *
 345    * @return string
 346    *   The processed string
 347    */
 348   public static function &replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 349     self::$_tokens['org']
 350       = array_merge(
 351         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 352         ['address', 'display_name', 'checksum', 'contact_id']
 353       );
 354
 355     $cv = NULL;
 356     foreach (self::$_tokens['org'] as $token) {
 357       // print "Getting token value for $token<br/><br/>";
 358       if ($token == '') {
 359         continue;
 360       }
 361
 362       // If the string doesn't contain this token, skip it.
 363
 364       if (!self::token_match('org', $token, $str)) {
 365         continue;
 366       }
 367
 368       // Construct value from $token and $contact
 369
 370       $value = NULL;
 371
 372       if ($cfID = CRM_Core_BAO_CustomField::getKeyID($token)) {
 373         // only generate cv if we need it
 374         if ($cv === NULL) {
 375           $cv = CRM_Core_BAO_CustomValue::getContactValues($org['contact_id']);
 376         }
 377         foreach ($cv as $cvFieldID => $value) {
 378           if ($cvFieldID == $cfID) {
 379             $value = CRM_Core_BAO_CustomField::displayValue($value, $cfID);
 380             break;
 381           }
 382         }
 383       }
 384       elseif ($token == 'checksum') {
 385         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 386         $value = "cs={$cs}";
 387       }
 388       elseif ($token == 'address') {
 389         // Build the location values array
 390
 391         $loc = [];
 392         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 393         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 394         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 395         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 396         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 397
 398         // Construct the address token
 399
 400         $value = CRM_Utils_Address::format($loc);
 401         if ($html) {
 402           $value = str_replace("\n", '<br />', $value);
 403         }
 404       }
 405       else {
 406         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 407       }
 408
 409       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 410     }
 411
 412     return $str;
 413   }
 414
 415   /**
 416    * Replace all mailing tokens in $str
 417    *
 418    * @param string $str
 419    *   The string with tokens to be replaced.
 420    * @param object $mailing
 421    *   The mailing BAO, or null for validation.
 422    * @param bool $html
 423    *   Replace tokens with HTML or plain text.
 424    *
 425    * @param null $knownTokens
 426    * @param bool $escapeSmarty
 427    *
 428    * @return string
 429    *   The processed string
 430    */
 431   public static function &replaceMailingTokens(
 432     $str,
 433     &$mailing,
 434     $html = FALSE,
 435     $knownTokens = NULL,
 436     $escapeSmarty = FALSE
 437   ) {
 438     $key = 'mailing';
 439     if (!$knownTokens || !isset($knownTokens[$key])) {
 440       return $str;
 441     }
 442
 443     $str = preg_replace_callback(
 444       self::tokenRegex($key),
 445       function ($matches) use (&$mailing, $escapeSmarty) {
 446         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 447       },
 448       $str
 449     );
 450     return $str;
 451   }
 452
 453   /**
 454    * @param $token
 455    * @param $mailing
 456    * @param bool $escapeSmarty
 457    *
 458    * @return string
 459    */
 460   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 461     $value = '';
 462     switch ($token) {
 463       // CRM-7663
 464
 465       case 'id':
 466         $value = $mailing ? $mailing->id : 'undefined';
 467         break;
 468
 469       case 'name':
 470         $value = $mailing ? $mailing->name : 'Mailing Name';
 471         break;
 472
 473       case 'group':
 474         $groups = $mailing ? $mailing->getGroupNames() : ['Mailing Groups'];
 475         $value = implode(', ', $groups);
 476         break;
 477
 478       case 'subject':
 479         $value = $mailing->subject;
 480         break;
 481
 482       case 'viewUrl':
 483         $mailingKey = $mailing->id;
 484         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 485           $mailingKey = $hash;
 486         }
 487         $value = CRM_Utils_System::url('civicrm/mailing/view',
 488           "reset=1&id={$mailingKey}",
 489           TRUE, NULL, FALSE, TRUE
 490         );
 491         break;
 492
 493       case 'editUrl':
 494       case 'scheduleUrl':
 495         // Note: editUrl and scheduleUrl used to be different, but now there's
 496         // one screen which can adapt based on permissions (in workflow mode).
 497         $value = CRM_Utils_System::url('civicrm/mailing/send',
 498           "reset=1&mid={$mailing->id}&continue=true",
 499           TRUE, NULL, FALSE, TRUE
 500         );
 501         break;
 502
 503       case 'html':
 504         $page = new CRM_Mailing_Page_View();
 505         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 506         break;
 507
 508       case 'approvalStatus':
 509         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 510         break;
 511
 512       case 'approvalNote':
 513         $value = $mailing->approval_note;
 514         break;
 515
 516       case 'approveUrl':
 517         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 518           "reset=1&mid={$mailing->id}",
 519           TRUE, NULL, FALSE, TRUE
 520         );
 521         break;
 522
 523       case 'creator':
 524         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 525         break;
 526
 527       case 'creatorEmail':
 528         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 529         break;
 530
 531       default:
 532         $value = "{mailing.$token}";
 533         break;
 534     }
 535
 536     if ($escapeSmarty) {
 537       $value = self::tokenEscapeSmarty($value);
 538     }
 539     return $value;
 540   }
 541
 542   /**
 543    * Replace all action tokens in $str
 544    *
 545    * @param string $str
 546    *   The string with tokens to be replaced.
 547    * @param array $addresses
 548    *   Assoc. array of VERP event addresses.
 549    * @param array $urls
 550    *   Assoc. array of action URLs.
 551    * @param bool $html
 552    *   Replace tokens with HTML or plain text.
 553    * @param array $knownTokens
 554    *   A list of tokens that are known to exist in the email body.
 555    *
 556    * @param bool $escapeSmarty
 557    *
 558    * @return string
 559    *   The processed string
 560    */
 561   public static function &replaceActionTokens(
 562     $str,
 563     &$addresses,
 564     &$urls,
 565     $html = FALSE,
 566     $knownTokens = NULL,
 567     $escapeSmarty = FALSE
 568   ) {
 569     $key = 'action';
 570     // here we intersect with the list of pre-configured valid tokens
 571     // so that we remove anything we do not recognize
 572     // I hope to move this step out of here soon and
 573     // then we will just iterate on a list of tokens that are passed to us
 574     if (!$knownTokens || empty($knownTokens[$key])) {
 575       return $str;
 576     }
 577
 578     $str = preg_replace_callback(
 579       self::tokenRegex($key),
 580       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 581         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 582       },
 583       $str
 584     );
 585     return $str;
 586   }
 587
 588   /**
 589    * @param $token
 590    * @param $addresses
 591    * @param $urls
 592    * @param bool $html
 593    * @param bool $escapeSmarty
 594    *
 595    * @return mixed|string
 596    */
 597   public static function getActionTokenReplacement(
 598     $token,
 599     &$addresses,
 600     &$urls,
 601     $html = FALSE,
 602     $escapeSmarty = FALSE
 603   ) {
 604     // If the token is an email action, use it.  Otherwise, find the
 605     // appropriate URL.
 606
 607     if (!in_array($token, self::$_tokens['action'])) {
 608       $value = "{action.$token}";
 609     }
 610     else {
 611       $value = CRM_Utils_Array::value($token, $addresses);
 612
 613       if ($value == NULL) {
 614         $value = CRM_Utils_Array::value($token, $urls);
 615       }
 616
 617       if ($value && $html) {
 618         // fix for CRM-2318
 619         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 620           $value = "mailto:$value";
 621         }
 622       }
 623       elseif ($value && !$html) {
 624         $value = str_replace('&amp;', '&', $value);
 625       }
 626     }
 627
 628     if ($escapeSmarty) {
 629       $value = self::tokenEscapeSmarty($value);
 630     }
 631     return $value;
 632   }
 633
 634   /**
 635    * Replace all the contact-level tokens in $str with information from
 636    * $contact.
 637    *
 638    * @param string $str
 639    *   The string with tokens to be replaced.
 640    * @param array $contact
 641    *   Associative array of contact properties.
 642    * @param bool $html
 643    *   Replace tokens with HTML or plain text.
 644    * @param array $knownTokens
 645    *   A list of tokens that are known to exist in the email body.
 646    * @param bool $returnBlankToken
 647    *   Return unevaluated token if value is null.
 648    *
 649    * @param bool $escapeSmarty
 650    *
 651    * @return string
 652    *   The processed string
 653    */
 654   public static function &replaceContactTokens(
 655     $str,
 656     &$contact,
 657     $html = FALSE,
 658     $knownTokens = NULL,
 659     $returnBlankToken = FALSE,
 660     $escapeSmarty = FALSE
 661   ) {
 662     // Refresh contact tokens in case they have changed. There is heavy caching
 663     // in exportable fields so there is no benefit in doing this conditionally.
 664     self::$_tokens['contact'] = array_merge(
 665       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 666       ['checksum', 'contact_id']
 667     );
 668
 669     $key = 'contact';
 670     // here we intersect with the list of pre-configured valid tokens
 671     // so that we remove anything we do not recognize
 672     // I hope to move this step out of here soon and
 673     // then we will just iterate on a list of tokens that are passed to us
 674     if (!$knownTokens || empty($knownTokens[$key])) {
 675       return $str;
 676     }
 677
 678     $str = preg_replace_callback(
 679       self::tokenRegex($key),
 680       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 681         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 682       },
 683       $str
 684     );
 685
 686     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 687     return $str;
 688   }
 689
 690   /**
 691    * @param $token
 692    * @param $contact
 693    * @param bool $html
 694    * @param bool $returnBlankToken
 695    * @param bool $escapeSmarty
 696    *
 697    * @return bool|mixed|null|string
 698    */
 699   public static function getContactTokenReplacement(
 700     $token,
 701     &$contact,
 702     $html = FALSE,
 703     $returnBlankToken = FALSE,
 704     $escapeSmarty = FALSE
 705   ) {
 706     if (self::$_tokens['contact'] == NULL) {
 707       /* This should come from UF */
 708
 709       self::$_tokens['contact']
 710         = array_merge(
 711           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 712           ['checksum', 'contact_id']
 713         );
 714     }
 715
 716     // Construct value from $token and $contact
 717
 718     $value = NULL;
 719     $noReplace = FALSE;
 720
 721     // Support legacy tokens
 722     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 723
 724     // check if the token we were passed is valid
 725     // we have to do this because this function is
 726     // called only when we find a token in the string
 727
 728     if (!in_array($token, self::$_tokens['contact'])) {
 729       $noReplace = TRUE;
 730     }
 731     elseif ($token == 'checksum') {
 732       $hash = CRM_Utils_Array::value('hash', $contact);
 733       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 734       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 735         NULL,
 736         NULL,
 737         $hash
 738       );
 739       $value = "cs={$cs}";
 740     }
 741     else {
 742       $value = CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 743
 744       // FIXME: for some pseudoconstants we get array ( 0 => id, 1 => label )
 745       if (is_array($value)) {
 746         $value = $value[1];
 747       }
 748       // Convert pseudoconstants using metadata
 749       elseif ($value && is_numeric($value)) {
 750         $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
 751         if (!empty($allFields[$token]['pseudoconstant'])) {
 752           $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
 753         }
 754       }
 755       elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
 756         $value = CRM_Utils_Date::customFormat($value);
 757       }
 758     }
 759
 760     if (!$html) {
 761       $value = str_replace('&amp;', '&', $value);
 762     }
 763
 764     // if null then return actual token
 765     if ($returnBlankToken && !$value) {
 766       $noReplace = TRUE;
 767     }
 768
 769     if ($noReplace) {
 770       $value = "{contact.$token}";
 771     }
 772
 773     if ($escapeSmarty
 774       && !($returnBlankToken && $noReplace)
 775     ) {
 776       // $returnBlankToken means the caller wants to do further attempts at
 777       // processing unreplaced tokens -- so don't escape them yet in this case.
 778       $value = self::tokenEscapeSmarty($value);
 779     }
 780
 781     return $value;
 782   }
 783
 784   /**
 785    * Replace all the hook tokens in $str with information from
 786    * $contact.
 787    *
 788    * @param string $str
 789    *   The string with tokens to be replaced.
 790    * @param array $contact
 791    *   Associative array of contact properties (including hook token values).
 792    * @param $categories
 793    * @param bool $html
 794    *   Replace tokens with HTML or plain text.
 795    *
 796    * @param bool $escapeSmarty
 797    *
 798    * @return string
 799    *   The processed string
 800    */
 801   public static function &replaceHookTokens(
 802     $str,
 803     &$contact,
 804     &$categories,
 805     $html = FALSE,
 806     $escapeSmarty = FALSE
 807   ) {
 808     foreach ($categories as $key) {
 809       $str = preg_replace_callback(
 810         self::tokenRegex($key),
 811         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 812           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 813         },
 814         $str
 815       );
 816     }
 817     return $str;
 818   }
 819
 820   /**
 821    * Parse html through Smarty resolving any smarty functions.
 822    * @param string $tokenHtml
 823    * @param array $entity
 824    * @param string $entityType
 825    * @return string
 826    *   html parsed through smarty
 827    */
 828   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 829     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 830       $smarty = CRM_Core_Smarty::singleton();
 831       // also add the tokens to the template
 832       $smarty->assign_by_ref($entityType, $entity);
 833       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 834     }
 835     return $tokenHtml;
 836   }
 837
 838   /**
 839    * @param $token
 840    * @param $contact
 841    * @param $category
 842    * @param bool $html
 843    * @param bool $escapeSmarty
 844    *
 845    * @return mixed|string
 846    */
 847   public static function getHookTokenReplacement(
 848     $token,
 849     &$contact,
 850     $category,
 851     $html = FALSE,
 852     $escapeSmarty = FALSE
 853   ) {
 854     $value = CRM_Utils_Array::value("{$category}.{$token}", $contact);
 855
 856     if ($value && !$html) {
 857       $value = str_replace('&amp;', '&', $value);
 858     }
 859
 860     if ($escapeSmarty) {
 861       $value = self::tokenEscapeSmarty($value);
 862     }
 863
 864     return $value;
 865   }
 866
 867   /**
 868    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 869    * for example {{token}}  or \{token}  will result in {token} in the final email
 870    *
 871    * this routine will remove the extra backslashes and braces
 872    *
 873    * @param $str ref to the string that will be scanned and modified
 874    */
 875   public static function unescapeTokens(&$str) {
 876     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 877   }
 878
 879   /**
 880    * Replace unsubscribe tokens.
 881    *
 882    * @param string $str
 883    *   The string with tokens to be replaced.
 884    * @param object $domain
 885    *   The domain BAO.
 886    * @param array $groups
 887    *   The groups (if any) being unsubscribed.
 888    * @param bool $html
 889    *   Replace tokens with html or plain text.
 890    * @param int $contact_id
 891    *   The contact ID.
 892    * @param string $hash The security hash of the unsub event
 893    *
 894    * @return string
 895    *   The processed string
 896    */
 897   public static function &replaceUnsubscribeTokens(
 898     $str,
 899     &$domain,
 900     &$groups,
 901     $html,
 902     $contact_id,
 903     $hash
 904   ) {
 905     if (self::token_match('unsubscribe', 'group', $str)) {
 906       if (!empty($groups)) {
 907         $config = CRM_Core_Config::singleton();
 908         $base = CRM_Utils_System::baseURL();
 909
 910         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 911         $dao = new CRM_Contact_DAO_Group();
 912         $dao->find();
 913         while ($dao->fetch()) {
 914           if (substr($dao->visibility, 0, 6) == 'Public') {
 915             $visibleGroups[] = $dao->id;
 916           }
 917         }
 918         $value = implode(', ', $groups);
 919         self::token_replace('unsubscribe', 'group', $value, $str);
 920       }
 921     }
 922     return $str;
 923   }
 924
 925   /**
 926    * Replace resubscribe tokens.
 927    *
 928    * @param string $str
 929    *   The string with tokens to be replaced.
 930    * @param object $domain
 931    *   The domain BAO.
 932    * @param array $groups
 933    *   The groups (if any) being resubscribed.
 934    * @param bool $html
 935    *   Replace tokens with html or plain text.
 936    * @param int $contact_id
 937    *   The contact ID.
 938    * @param string $hash The security hash of the resub event
 939    *
 940    * @return string
 941    *   The processed string
 942    */
 943   public static function &replaceResubscribeTokens(
 944     $str, &$domain, &$groups, $html,
 945     $contact_id, $hash
 946   ) {
 947     if (self::token_match('resubscribe', 'group', $str)) {
 948       if (!empty($groups)) {
 949         $value = implode(', ', $groups);
 950         self::token_replace('resubscribe', 'group', $value, $str);
 951       }
 952     }
 953     return $str;
 954   }
 955
 956   /**
 957    * Replace subscription-confirmation-request tokens
 958    *
 959    * @param string $str
 960    *   The string with tokens to be replaced.
 961    * @param string $group
 962    *   The name of the group being subscribed.
 963    * @param $url
 964    * @param bool $html
 965    *   Replace tokens with html or plain text.
 966    *
 967    * @return string
 968    *   The processed string
 969    */
 970   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 971     if (self::token_match('subscribe', 'group', $str)) {
 972       self::token_replace('subscribe', 'group', $group, $str);
 973     }
 974     if (self::token_match('subscribe', 'url', $str)) {
 975       self::token_replace('subscribe', 'url', $url, $str);
 976     }
 977     return $str;
 978   }
 979
 980   /**
 981    * Replace subscription-invitation tokens
 982    *
 983    * @param string $str
 984    *   The string with tokens to be replaced.
 985    *
 986    * @return string
 987    *   The processed string
 988    */
 989   public static function &replaceSubscribeInviteTokens($str) {
 990     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 991       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 992         'reset=1',
 993         TRUE, NULL, FALSE, TRUE
 994       );
 995       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 996     }
 997
 998     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 999       foreach ($matches as $key => $value) {
1000         $gid = substr($value, 21, -1);
1001         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
1002           "reset=1&gid={$gid}",
1003           TRUE, NULL, FALSE, TRUE
1004         );
1005         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
1006       }
1007     }
1008
1009     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
1010       foreach ($matches as $key => $value) {
1011         $gid = substr($value, 18, -1);
1012         $config = CRM_Core_Config::singleton();
1013         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
1014         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
1015         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
1016         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1017       }
1018     }
1019     return $str;
1020   }
1021
1022   /**
1023    * Replace welcome/confirmation tokens
1024    *
1025    * @param string $str
1026    *   The string with tokens to be replaced.
1027    * @param string $group
1028    *   The name of the group being subscribed.
1029    * @param bool $html
1030    *   Replace tokens with html or plain text.
1031    *
1032    * @return string
1033    *   The processed string
1034    */
1035   public static function &replaceWelcomeTokens($str, $group, $html) {
1036     if (self::token_match('welcome', 'group', $str)) {
1037       self::token_replace('welcome', 'group', $group, $str);
1038     }
1039     return $str;
1040   }
1041
1042   /**
1043    * Find unprocessed tokens (call this last)
1044    *
1045    * @param string $str
1046    *   The string to search.
1047    *
1048    * @return array
1049    *   Array of tokens that weren't replaced
1050    */
1051   public static function &unmatchedTokens(&$str) {
1052     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1053     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1054     return $match[1];
1055   }
1056
1057   /**
1058    * Find and replace tokens for each component.
1059    *
1060    * @param string $str
1061    *   The string to search.
1062    * @param array $contact
1063    *   Associative array of contact properties.
1064    * @param array $components
1065    *   A list of tokens that are known to exist in the email body.
1066    *
1067    * @param bool $escapeSmarty
1068    * @param bool $returnEmptyToken
1069    *
1070    * @return string
1071    *   The processed string
1072    */
1073   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1074     if (!is_array($components) || empty($contact)) {
1075       return $str;
1076     }
1077
1078     foreach ($components as $name => $tokens) {
1079       if (!is_array($tokens) || empty($tokens)) {
1080         continue;
1081       }
1082
1083       foreach ($tokens as $token) {
1084         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1085           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1086         }
1087         elseif (!$returnEmptyToken) {
1088           //replacing empty token
1089           self::token_replace($name, $token, "", $str, $escapeSmarty);
1090         }
1091       }
1092     }
1093     return $str;
1094   }
1095
1096   /**
1097    * Get array of string tokens.
1098    *
1099    * @param string $string
1100    *   The input string to parse for tokens.
1101    *
1102    * @return array
1103    *   array of tokens mentioned in field
1104    */
1105   public static function getTokens($string) {
1106     $matches = [];
1107     $tokens = [];
1108     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1109       $string,
1110       $matches,
1111       PREG_PATTERN_ORDER
1112     );
1113
1114     if ($matches[1]) {
1115       foreach ($matches[1] as $token) {
1116         list($type, $name) = preg_split('/\./', $token, 2);
1117         if ($name && $type) {
1118           if (!isset($tokens[$type])) {
1119             $tokens[$type] = [];
1120           }
1121           $tokens[$type][] = $name;
1122         }
1123       }
1124     }
1125     return $tokens;
1126   }
1127
1128   /**
1129    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1130    * and getTokens appears to be historical rather than intentional and should be reviewed
1131    * @param $string
1132    * @return array
1133    *   fields to pass in as return properties when populating token
1134    */
1135   public static function getReturnProperties(&$string) {
1136     $returnProperties = [];
1137     $matches = [];
1138     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1139       $string,
1140       $matches,
1141       PREG_PATTERN_ORDER
1142     );
1143     if ($matches[1]) {
1144       foreach ($matches[1] as $token) {
1145         list($type, $name) = preg_split('/\./', $token, 2);
1146         if ($name) {
1147           $returnProperties["{$name}"] = 1;
1148         }
1149       }
1150     }
1151
1152     return $returnProperties;
1153   }
1154
1155   /**
1156    * Gives required details of contacts in an indexed array format so we
1157    * can iterate in a nice loop and do token evaluation
1158    *
1159    * @param array $contactIDs
1160    * @param array $returnProperties
1161    *   Of required properties.
1162    * @param bool $skipOnHold Don't return on_hold contact info also.
1163    *   Don't return on_hold contact info also.
1164    * @param bool $skipDeceased Don't return deceased contact info.
1165    *   Don't return deceased contact info.
1166    * @param array $extraParams
1167    *   Extra params.
1168    * @param array $tokens
1169    *   The list of tokens we've extracted from the content.
1170    * @param null $className
1171    * @param int $jobID
1172    *   The mailing list jobID - this is a legacy param.
1173    *
1174    * @return array
1175    */
1176   public static function getTokenDetails(
1177     $contactIDs,
1178     $returnProperties = NULL,
1179     $skipOnHold = TRUE,
1180     $skipDeceased = TRUE,
1181     $extraParams = NULL,
1182     $tokens = [],
1183     $className = NULL,
1184     $jobID = NULL
1185   ) {
1186
1187     $params = [];
1188     foreach ($contactIDs as $contactID) {
1189       $params[] = [
1190         CRM_Core_Form::CB_PREFIX . $contactID,
1191         '=',
1192         1,
1193         0,
1194         0,
1195       ];
1196     }
1197
1198     // fix for CRM-2613
1199     if ($skipDeceased) {
1200       $params[] = ['is_deceased', '=', 0, 0, 0];
1201     }
1202
1203     //fix for CRM-3798
1204     if ($skipOnHold) {
1205       $params[] = ['on_hold', '=', 0, 0, 0];
1206     }
1207
1208     if ($extraParams) {
1209       $params = array_merge($params, $extraParams);
1210     }
1211
1212     // if return properties are not passed then get all return properties
1213     if (empty($returnProperties)) {
1214       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1215         ['display_name', 'checksum', 'contact_id']
1216       );
1217       foreach ($fields as $val) {
1218         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1219         // with contacts and are expensive to resolve.
1220         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1221         // and upstream calls to this should populate return properties.
1222         $unavailableFields = ['group', 'tag'];
1223         if (!in_array($val, $unavailableFields)) {
1224           $returnProperties[$val] = 1;
1225         }
1226       }
1227     }
1228
1229     $custom = [];
1230     foreach ($returnProperties as $name => $dontCare) {
1231       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1232       if ($cfID) {
1233         $custom[] = $cfID;
1234       }
1235     }
1236
1237     $details = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1238
1239     $contactDetails = &$details[0];
1240
1241     foreach ($contactIDs as $contactID) {
1242       if (array_key_exists($contactID, $contactDetails)) {
1243         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1244         ) {
1245           $communicationPreferences = [];
1246           foreach ($contactDetails[$contactID]['preferred_communication_method'] as $val) {
1247             if ($val) {
1248               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1249             }
1250           }
1251           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1252         }
1253
1254         foreach ($custom as $cfID) {
1255           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1256             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1257           }
1258         }
1259
1260         // special case for greeting replacement
1261         foreach ([
1262           'email_greeting',
1263           'postal_greeting',
1264           'addressee',
1265         ] as $val) {
1266           if (!empty($contactDetails[$contactID][$val])) {
1267             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1268           }
1269         }
1270       }
1271     }
1272
1273     // also call a hook and get token details
1274     CRM_Utils_Hook::tokenValues($details[0],
1275       $contactIDs,
1276       $jobID,
1277       $tokens,
1278       $className
1279     );
1280     return $details;
1281   }
1282
1283   /**
1284    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1285    * specific tokens to be rendered
1286    *
1287    * @param array $contactIDs
1288    *   This should always be array(0) or its not anonymous - left to keep signature same.
1289    *   as main fn
1290    * @param string $returnProperties
1291    * @param bool $skipOnHold
1292    * @param bool $skipDeceased
1293    * @param string $extraParams
1294    * @param array $tokens
1295    * @param string $className
1296    *   Sent as context to the hook.
1297    * @param string $jobID
1298    * @return array
1299    *   contactDetails with hooks swapped out
1300    */
1301   public static function getAnonymousTokenDetails($contactIDs = [
1302     0,
1303   ],
1304                                            $returnProperties = NULL,
1305                                            $skipOnHold = TRUE,
1306                                            $skipDeceased = TRUE,
1307                                            $extraParams = NULL,
1308                                            $tokens = [],
1309                                            $className = NULL,
1310                                            $jobID = NULL) {
1311     $details = [0 => []];
1312     // also call a hook and get token details
1313     CRM_Utils_Hook::tokenValues($details[0],
1314       $contactIDs,
1315       $jobID,
1316       $tokens,
1317       $className
1318     );
1319     return $details;
1320   }
1321
1322   /**
1323    * Get Membership Token Details.
1324    * @param array $membershipIDs
1325    *   Array of membership IDS.
1326    */
1327   public static function getMembershipTokenDetails($membershipIDs) {
1328     $memberships = civicrm_api3('membership', 'get', [
1329       'options' => ['limit' => 0],
1330       'membership_id' => ['IN' => (array) $membershipIDs],
1331     ]);
1332     return $memberships['values'];
1333   }
1334
1335   /**
1336    * Replace existing greeting tokens in message/subject.
1337    *
1338    * This function operates by reference, modifying the first parameter. Other
1339    * methods for token replacement in this class return the modified string.
1340    * This leads to inconsistency in how these methods must be applied.
1341    *
1342    * @TODO Remove that inconsistency in usage.
1343    *
1344    * ::replaceContactTokens() may need to be called after this method, to
1345    * replace tokens supplied from this method.
1346    *
1347    * @param string $tokenString
1348    * @param array $contactDetails
1349    * @param int $contactId
1350    * @param string $className
1351    * @param bool $escapeSmarty
1352    */
1353   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1354
1355     if (!$contactDetails && !$contactId) {
1356       return;
1357     }
1358
1359     // check if there are any tokens
1360     $greetingTokens = self::getTokens($tokenString);
1361
1362     if (!empty($greetingTokens)) {
1363       // first use the existing contact object for token replacement
1364       if (!empty($contactDetails)) {
1365         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1366       }
1367
1368       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1369       // check if there are any unevaluated tokens
1370       $greetingTokens = self::getTokens($tokenString);
1371
1372       // $greetingTokens not empty, means there are few tokens which are not
1373       // evaluated, like custom data etc
1374       // so retrieve it from database
1375       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1376         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1377         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1378         $contactParams = ['contact_id' => $contactId];
1379
1380         $greetingDetails = self::getTokenDetails($contactParams,
1381           $greetingsReturnProperties,
1382           FALSE, FALSE, NULL,
1383           $greetingTokens,
1384           $className
1385         );
1386
1387         // again replace tokens
1388         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1389           $greetingDetails,
1390           TRUE,
1391           $greetingTokens,
1392           TRUE,
1393           $escapeSmarty
1394         );
1395       }
1396
1397       // check if there are still any unevaluated tokens
1398       $remainingTokens = self::getTokens($tokenString);
1399
1400       // $greetingTokens not empty, there are customized or hook tokens to replace
1401       if (!empty($remainingTokens)) {
1402         // Fill the return properties array
1403         $greetingTokens = $remainingTokens;
1404         reset($greetingTokens);
1405         $greetingsReturnProperties = [];
1406         foreach ($greetingTokens as $value) {
1407           $props = array_flip($value);
1408           $props = array_fill_keys(array_keys($props), 1);
1409           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1410         }
1411         $contactParams = ['contact_id' => $contactId];
1412         $greetingDetails = self::getTokenDetails($contactParams,
1413           $greetingsReturnProperties,
1414           FALSE, FALSE, NULL,
1415           $greetingTokens,
1416           $className
1417         );
1418         // Prepare variables for calling replaceHookTokens
1419         $categories = array_keys($greetingTokens);
1420         list($contact) = $greetingDetails;
1421         // Replace tokens defined in Hooks.
1422         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1423       }
1424     }
1425   }
1426
1427   /**
1428    * At this point, $contactDetails has loaded the contact from the DAO. Any
1429    * (non-custom) missing fields are null.  By removing them, we can avoid
1430    * expensive calls to CRM_Contact_BAO_Query.
1431    *
1432    * @param string $tokenString
1433    * @param array $contactDetails
1434    * @param array $greetingTokens
1435    */
1436   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1437
1438     // Only applies to contact tokens
1439     if (!array_key_exists('contact', $greetingTokens)) {
1440       return;
1441     }
1442
1443     $greetingTokensOriginal = $greetingTokens;
1444     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1445     // Sometimes contactDetails are in a multidemensional array, sometimes a
1446     // single-dimension array.
1447     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1448       $contactDetails = current($contactDetails[0]);
1449     }
1450     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1451
1452     // Handle legacy tokens
1453     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1454       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1455         $nullFields[] = $oldToken;
1456       }
1457     }
1458
1459     // Remove null contact fields from $greetingTokens
1460     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1461
1462     // Also remove them from $tokenString
1463     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1464     // Handle legacy tokens again, sigh
1465     if (!empty($removedTokens)) {
1466       foreach ($removedTokens as $token) {
1467         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1468           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1469         }
1470       }
1471       foreach ($removedTokens as $token) {
1472         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1473       }
1474     }
1475   }
1476
1477   /**
1478    * @param $tokens
1479    *
1480    * @return array
1481    */
1482   public static function flattenTokens(&$tokens) {
1483     $flattenTokens = [];
1484
1485     foreach ([
1486       'html',
1487       'text',
1488       'subject',
1489     ] as $prop) {
1490       if (!isset($tokens[$prop])) {
1491         continue;
1492       }
1493       foreach ($tokens[$prop] as $type => $names) {
1494         if (!isset($flattenTokens[$type])) {
1495           $flattenTokens[$type] = [];
1496         }
1497         foreach ($names as $name) {
1498           $flattenTokens[$type][$name] = 1;
1499         }
1500       }
1501     }
1502
1503     return $flattenTokens;
1504   }
1505
1506   /**
1507    * Replace all user tokens in $str
1508    *
1509    * @param string $str
1510    *   The string with tokens to be replaced.
1511    *
1512    * @param null $knownTokens
1513    * @param bool $escapeSmarty
1514    *
1515    * @return string
1516    *   The processed string
1517    */
1518   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1519     $key = 'user';
1520     if (!$knownTokens ||
1521       !isset($knownTokens[$key])
1522     ) {
1523       return $str;
1524     }
1525
1526     $str = preg_replace_callback(
1527       self::tokenRegex($key),
1528       function ($matches) use ($escapeSmarty) {
1529         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1530       },
1531       $str
1532     );
1533     return $str;
1534   }
1535
1536   /**
1537    * @param $token
1538    * @param bool $escapeSmarty
1539    *
1540    * @return string
1541    */
1542   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1543     $value = '';
1544
1545     list($objectName, $objectValue) = explode('-', $token, 2);
1546
1547     switch ($objectName) {
1548       case 'permission':
1549         $value = CRM_Core_Permission::permissionEmails($objectValue);
1550         break;
1551
1552       case 'role':
1553         $value = CRM_Core_Permission::roleEmails($objectValue);
1554         break;
1555     }
1556
1557     if ($escapeSmarty) {
1558       $value = self::tokenEscapeSmarty($value);
1559     }
1560
1561     return $value;
1562   }
1563
1564   protected static function _buildContributionTokens() {
1565     $key = 'contribution';
1566     if (self::$_tokens[$key] == NULL) {
1567       self::$_tokens[$key] = array_keys(array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1568         ['campaign', 'financial_type'],
1569         self::getCustomFieldTokens('Contribution')
1570       ));
1571     }
1572   }
1573
1574   /**
1575    * Store membership tokens on the static _tokens array.
1576    */
1577   protected static function _buildMembershipTokens() {
1578     $key = 'membership';
1579     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1580       $membershipTokens = [];
1581       $tokens = CRM_Core_SelectValues::membershipTokens();
1582       foreach ($tokens as $token => $dontCare) {
1583         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1584       }
1585       self::$_tokens[$key] = $membershipTokens;
1586     }
1587   }
1588
1589   /**
1590    * Replace tokens for an entity.
1591    * @param string $entity
1592    * @param array $entityArray
1593    *   (e.g. in format from api).
1594    * @param string $str
1595    *   String to replace in.
1596    * @param array $knownTokens
1597    *   Array of tokens present.
1598    * @param bool $escapeSmarty
1599    * @return string
1600    *   string with replacements made
1601    */
1602   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1603     if (!$knownTokens || empty($knownTokens[$entity])) {
1604       return $str;
1605     }
1606
1607     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1608     $fn = is_callable(['CRM_Utils_Token', $fn]) ? $fn : 'getApiTokenReplacement';
1609     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1610     foreach ($knownTokens[$entity] as $token) {
1611       $replacement = self::$fn($entity, $token, $entityArray);
1612       if ($escapeSmarty) {
1613         $replacement = self::tokenEscapeSmarty($replacement);
1614       }
1615       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1616     }
1617     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1618   }
1619
1620   /**
1621    * @param int $caseId
1622    * @param int $str
1623    * @param array $knownTokens
1624    * @param bool $escapeSmarty
1625    * @return string
1626    * @throws \CiviCRM_API3_Exception
1627    */
1628   public static function replaceCaseTokens($caseId, $str, $knownTokens = [], $escapeSmarty = FALSE) {
1629     if (!$knownTokens || empty($knownTokens['case'])) {
1630       return $str;
1631     }
1632     $case = civicrm_api3('case', 'getsingle', ['id' => $caseId]);
1633     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1634   }
1635
1636   /**
1637    * Generic function for formatting token replacement for an api field
1638    *
1639    * @param string $entity
1640    * @param string $token
1641    * @param array $entityArray
1642    * @return string
1643    * @throws \CiviCRM_API3_Exception
1644    */
1645   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1646     if (!isset($entityArray[$token])) {
1647       return '';
1648     }
1649     $field = civicrm_api3($entity, 'getfield', ['action' => 'get', 'name' => $token, 'get_options' => 'get']);
1650     $field = $field['values'];
1651     $fieldType = CRM_Utils_Array::value('type', $field);
1652     // Boolean fields
1653     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1654       $field['options'] = [ts('No'), ts('Yes')];
1655     }
1656     // Match pseudoconstants
1657     if (!empty($field['options'])) {
1658       $ret = [];
1659       foreach ((array) $entityArray[$token] as $val) {
1660         $ret[] = $field['options'][$val];
1661       }
1662       return implode(', ', $ret);
1663     }
1664     // Format date fields
1665     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1666       return CRM_Utils_Date::customFormat($entityArray[$token]);
1667     }
1668     return implode(', ', (array) $entityArray[$token]);
1669   }
1670
1671   /**
1672    * Replace Contribution tokens in html.
1673    *
1674    * @param string $str
1675    * @param array $contribution
1676    * @param bool|string $html
1677    * @param string $knownTokens
1678    * @param bool|string $escapeSmarty
1679    *
1680    * @return mixed
1681    */
1682   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1683     $key = 'contribution';
1684     if (!$knownTokens || !CRM_Utils_Array::value($key, $knownTokens)) {
1685       //early return
1686       return $str;
1687     }
1688     self::_buildContributionTokens();
1689
1690     // here we intersect with the list of pre-configured valid tokens
1691     // so that we remove anything we do not recognize
1692     // I hope to move this step out of here soon and
1693     // then we will just iterate on a list of tokens that are passed to us
1694
1695     $str = preg_replace_callback(
1696       self::tokenRegex($key),
1697       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1698         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1699       },
1700       $str
1701     );
1702
1703     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1704     return $str;
1705   }
1706
1707   /**
1708    * We have a situation where we are rendering more than one token in each field because we are combining
1709    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1710    *
1711    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1712    * it applies (other tokens are OK including date fields)
1713    *
1714    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1715    * does - but the fields is already being formatted through a date function
1716    *
1717    * @param string $separator
1718    * @param string $str
1719    * @param array $contribution
1720    * @param bool|string $html
1721    * @param string $knownTokens
1722    * @param bool|string $escapeSmarty
1723    *
1724    * @return string
1725    */
1726   public static function replaceMultipleContributionTokens($separator, $str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1727     if (empty($knownTokens['contribution'])) {
1728       return $str;
1729     }
1730
1731     if (in_array('receive_date', $knownTokens['contribution'])) {
1732       $formattedDates = [];
1733       $dates = explode($separator, $contribution['receive_date']);
1734       foreach ($dates as $date) {
1735         $formattedDates[] = CRM_Utils_Date::customFormat($date, NULL, ['j', 'm', 'Y']);
1736       }
1737       $str = str_replace("{contribution.receive_date}", implode($separator, $formattedDates), $str);
1738       unset($knownTokens['contribution']['receive_date']);
1739     }
1740     return self::replaceContributionTokens($str, $contribution, $html, $knownTokens, $escapeSmarty);
1741   }
1742
1743   /**
1744    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1745    * - this is used by the pdfLetter task from membership search
1746    * @param string $entity
1747    *   should always be "membership"
1748    * @param string $token
1749    *   field name
1750    * @param array $membership
1751    *   An api result array for a single membership.
1752    * @return string token replacement
1753    */
1754   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1755     self::_buildMembershipTokens();
1756     switch ($token) {
1757       case 'type':
1758         $value = $membership['membership_name'];
1759         break;
1760
1761       case 'status':
1762         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1763         $value = $statuses[$membership['status_id']];
1764         break;
1765
1766       case 'fee':
1767         try {
1768           $value = civicrm_api3('membership_type', 'getvalue', [
1769             'id' => $membership['membership_type_id'],
1770             'return' => 'minimum_fee',
1771           ]);
1772           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1773         }
1774         catch (CiviCRM_API3_Exception $e) {
1775           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1776           // api handles NULL (4.4)
1777           $value = 0;
1778         }
1779         break;
1780
1781       default:
1782         if (in_array($token, self::$_tokens[$entity])) {
1783           $value = $membership[$token];
1784           if (CRM_Utils_String::endsWith($token, '_date')) {
1785             $value = CRM_Utils_Date::customFormat($value);
1786           }
1787         }
1788         else {
1789           // ie unchanged
1790           $value = "{$entity}.{$token}";
1791         }
1792         break;
1793     }
1794
1795     return $value;
1796   }
1797
1798   /**
1799    * @param $token
1800    * @param $contribution
1801    * @param bool $html
1802    * @param bool $escapeSmarty
1803    *
1804    * @return mixed|string
1805    */
1806   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1807     self::_buildContributionTokens();
1808
1809     switch ($token) {
1810       case 'total_amount':
1811       case 'net_amount':
1812       case 'fee_amount':
1813       case 'non_deductible_amount':
1814         $value = CRM_Utils_Money::format(CRM_Utils_Array::retrieveValueRecursive($contribution, $token));
1815         break;
1816
1817       case 'receive_date':
1818         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1819         $value = CRM_Utils_Date::customFormat($value, NULL, ['j', 'm', 'Y']);
1820         break;
1821
1822       default:
1823         if (!in_array($token, self::$_tokens['contribution'])) {
1824           $value = "{contribution.$token}";
1825         }
1826         else {
1827           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1828         }
1829         break;
1830     }
1831
1832     if ($escapeSmarty) {
1833       $value = self::tokenEscapeSmarty($value);
1834     }
1835     return $value;
1836   }
1837
1838   /**
1839    * @return array
1840    *   [legacy_token => new_token]
1841    */
1842   public static function legacyContactTokens() {
1843     return [
1844       'individual_prefix' => 'prefix_id',
1845       'individual_suffix' => 'suffix_id',
1846       'gender' => 'gender_id',
1847       'communication_style' => 'communication_style_id',
1848     ];
1849   }
1850
1851   /**
1852    * Get all custom field tokens of $entity
1853    *
1854    * @param string $entity
1855    * @param bool $usedForTokenWidget
1856    *
1857    * @return array
1858    *   return custom field tokens in array('custom_N' => 'label') format
1859    */
1860   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1861     $customTokens = [];
1862     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1863     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1864       $customTokens[sprintf($tokenName, $id)] = $info['label'];
1865     }
1866
1867     return $customTokens;
1868   }
1869
1870   /**
1871    * Formats a token list for the select2 widget
1872    *
1873    * @param $tokens
1874    * @return array
1875    */
1876   public static function formatTokensForDisplay($tokens) {
1877     $sorted = $output = [];
1878
1879     // Sort in ascending order by ignoring word case
1880     natcasesort($tokens);
1881
1882     // Attempt to place tokens into optgroups
1883     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1884     foreach ($tokens as $k => $v) {
1885       // Check to see if this token is already in a group e.g. for custom fields
1886       $split = explode(' :: ', $v);
1887       if (!empty($split[1])) {
1888         $sorted[$split[1]][] = ['id' => $k, 'text' => $split[0]];
1889       }
1890       // Group by entity
1891       else {
1892         $split = explode('.', trim($k, '{}'));
1893         if (isset($split[1])) {
1894           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucfirst($split[0]);
1895         }
1896         else {
1897           $entity = 'Contact';
1898         }
1899         $sorted[ts($entity)][] = ['id' => $k, 'text' => $v];
1900       }
1901     }
1902
1903     ksort($sorted);
1904     foreach ($sorted as $k => $v) {
1905       $output[] = ['text' => $k, 'children' => $v];
1906     }
1907
1908     return $output;
1909   }
1910
1911 }