CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2016                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2016
  32  */
  33
  34 /**
  35  * Class to abstract token replacement.
  36  */
  37 class CRM_Utils_Token {
  38   static $_requiredTokens = NULL;
  39
  40   static $_tokens = array(
  41     'action' => array(
  42       'forward',
  43       'optOut',
  44       'optOutUrl',
  45       'reply',
  46       'unsubscribe',
  47       'unsubscribeUrl',
  48       'resubscribe',
  49       'resubscribeUrl',
  50       'subscribeUrl',
  51     ),
  52     'mailing' => array(
  53       'id',
  54       'name',
  55       'group',
  56       'subject',
  57       'viewUrl',
  58       'editUrl',
  59       'scheduleUrl',
  60       'approvalStatus',
  61       'approvalNote',
  62       'approveUrl',
  63       'creator',
  64       'creatorEmail',
  65     ),
  66     'user' => array(
  67       // we extract the stuff after the role / permission and return the
  68       // civicrm email addresses of all users with that role / permission
  69       // useful with rules integration
  70       'permission:',
  71       'role:',
  72     ),
  73     // populate this dynamically
  74     'contact' => NULL,
  75     // populate this dynamically
  76     'contribution' => NULL,
  77     'domain' => array(
  78       'name',
  79       'phone',
  80       'address',
  81       'email',
  82       'id',
  83       'description',
  84     ),
  85     'subscribe' => array('group'),
  86     'unsubscribe' => array('group'),
  87     'resubscribe' => array('group'),
  88     'welcome' => array('group'),
  89   );
  90
  91
  92   /**
  93    * @return array
  94    */
  95   public static function getRequiredTokens() {
  96     if (self::$_requiredTokens == NULL) {
  97       self::$_requiredTokens = array(
  98         'domain.address' => ts("Domain address - displays your organization's postal address."),
  99         'action.optOutUrl or action.unsubscribeUrl' => array(
 100           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
 101           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
 102           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
 103           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
 104         ),
 105       );
 106     }
 107     return self::$_requiredTokens;
 108   }
 109
 110   /**
 111    * Check a string (mailing body) for required tokens.
 112    *
 113    * @param string $str
 114    *   The message.
 115    *
 116    * @return bool|array
 117    *    true if all required tokens are found,
 118    *    else an array of the missing tokens
 119    */
 120   public static function requiredTokens(&$str) {
 121     $requiredTokens = self::getRequiredTokens();
 122
 123     $missing = array();
 124     foreach ($requiredTokens as $token => $value) {
 125       if (!is_array($value)) {
 126         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 127           $missing[$token] = $value;
 128         }
 129       }
 130       else {
 131         $present = FALSE;
 132         $desc = NULL;
 133         foreach ($value as $t => $d) {
 134           $desc = $d;
 135           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 136             $present = TRUE;
 137           }
 138         }
 139         if (!$present) {
 140           $missing[$token] = $desc;
 141         }
 142       }
 143     }
 144
 145     if (empty($missing)) {
 146       return TRUE;
 147     }
 148     return $missing;
 149   }
 150
 151   /**
 152    * Wrapper for token matching.
 153    *
 154    * @param string $type
 155    *   The token type (domain,mailing,contact,action).
 156    * @param string $var
 157    *   The token variable.
 158    * @param string $str
 159    *   The string to search.
 160    *
 161    * @return bool
 162    *   Was there a match
 163    */
 164   public static function token_match($type, $var, &$str) {
 165     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 166     return preg_match("/(^|[^\{])$token/", $str);
 167   }
 168
 169   /**
 170    * Wrapper for token replacing.
 171    *
 172    * @param string $type
 173    *   The token type.
 174    * @param string $var
 175    *   The token variable.
 176    * @param string $value
 177    *   The value to substitute for the token.
 178    * @param string (reference) $str The string to replace in
 179    *
 180    * @param bool $escapeSmarty
 181    *
 182    * @return string
 183    *   The processed string
 184    */
 185   public static function &token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 186     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 187     if (!$value) {
 188       $value = '$3';
 189     }
 190     if ($escapeSmarty) {
 191       $value = self::tokenEscapeSmarty($value);
 192     }
 193     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 194     return $str;
 195   }
 196
 197   /**
 198    * Get< the regex for token replacement
 199    *
 200    * @param string $token_type
 201    *   A string indicating the the type of token to be used in the expression.
 202    *
 203    * @return string
 204    *   regular expression sutiable for using in preg_replace
 205    */
 206   private static function tokenRegex($token_type) {
 207     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(\-[\w\s]+)?)\}(?!\})/';
 208   }
 209
 210   /**
 211    * Escape the string so a malicious user cannot inject smarty code into the template.
 212    *
 213    * @param string $string
 214    *   A string that needs to be escaped from smarty parsing.
 215    *
 216    * @return string
 217    *   the escaped string
 218    */
 219   public static function tokenEscapeSmarty($string) {
 220     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 221     return preg_replace(array('/{/', '/(?<!{ldelim)}/'), array('{ldelim}', '{rdelim}'), $string);
 222   }
 223
 224   /**
 225    * Replace all the domain-level tokens in $str
 226    *
 227    * @param string $str
 228    *   The string with tokens to be replaced.
 229    * @param object $domain
 230    *   The domain BAO.
 231    * @param bool $html
 232    *   Replace tokens with HTML or plain text.
 233    *
 234    * @param null $knownTokens
 235    * @param bool $escapeSmarty
 236    *
 237    * @return string
 238    *   The processed string
 239    */
 240   public static function &replaceDomainTokens(
 241     $str,
 242     &$domain,
 243     $html = FALSE,
 244     $knownTokens = NULL,
 245     $escapeSmarty = FALSE
 246   ) {
 247     $key = 'domain';
 248     if (
 249       !$knownTokens || empty($knownTokens[$key])
 250     ) {
 251       return $str;
 252     }
 253
 254     $str = preg_replace_callback(
 255       self::tokenRegex($key),
 256       function ($matches) use (&$domain, $html, $escapeSmarty) {
 257         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 258       },
 259       $str
 260     );
 261     return $str;
 262   }
 263
 264   /**
 265    * @param $token
 266    * @param $domain
 267    * @param bool $html
 268    * @param bool $escapeSmarty
 269    *
 270    * @return mixed|null|string
 271    */
 272   public static function getDomainTokenReplacement($token, &$domain, $html = FALSE, $escapeSmarty = FALSE) {
 273     // check if the token we were passed is valid
 274     // we have to do this because this function is
 275     // called only when we find a token in the string
 276
 277     $loc = &$domain->getLocationValues();
 278
 279     if (!in_array($token, self::$_tokens['domain'])) {
 280       $value = "{domain.$token}";
 281     }
 282     elseif ($token == 'address') {
 283       static $addressCache = array();
 284
 285       $cache_key = $html ? 'address-html' : 'address-text';
 286       if (array_key_exists($cache_key, $addressCache)) {
 287         return $addressCache[$cache_key];
 288       }
 289
 290       $value = NULL;
 291       // Construct the address token
 292
 293       if (!empty($loc[$token])) {
 294         if ($html) {
 295           $value = $loc[$token][1]['display'];
 296           $value = str_replace("\n", '<br />', $value);
 297         }
 298         else {
 299           $value = $loc[$token][1]['display_text'];
 300         }
 301         $addressCache[$cache_key] = $value;
 302       }
 303     }
 304     elseif ($token == 'name' || $token == 'id' || $token == 'description') {
 305       $value = $domain->$token;
 306     }
 307     elseif ($token == 'phone' || $token == 'email') {
 308       // Construct the phone and email tokens
 309
 310       $value = NULL;
 311       if (!empty($loc[$token])) {
 312         foreach ($loc[$token] as $index => $entity) {
 313           $value = $entity[$token];
 314           break;
 315         }
 316       }
 317     }
 318
 319     if ($escapeSmarty) {
 320       $value = self::tokenEscapeSmarty($value);
 321     }
 322
 323     return $value;
 324   }
 325
 326   /**
 327    * Replace all the org-level tokens in $str
 328    *
 329    * @fixme: This function appears to be broken, as it depends on
 330    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 331    * Marking as deprecated until this is fixed
 332    * @deprecated
 333    *
 334    * @param string $str
 335    *   The string with tokens to be replaced.
 336    * @param object $org
 337    *   Associative array of org properties.
 338    * @param bool $html
 339    *   Replace tokens with HTML or plain text.
 340    *
 341    * @param bool $escapeSmarty
 342    *
 343    * @return string
 344    *   The processed string
 345    */
 346   public static function &replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 347     self::$_tokens['org']
 348       = array_merge(
 349         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 350         array('address', 'display_name', 'checksum', 'contact_id')
 351       );
 352
 353     $cv = NULL;
 354     foreach (self::$_tokens['org'] as $token) {
 355       // print "Getting token value for $token<br/><br/>";
 356       if ($token == '') {
 357         continue;
 358       }
 359
 360       // If the string doesn't contain this token, skip it.
 361
 362       if (!self::token_match('org', $token, $str)) {
 363         continue;
 364       }
 365
 366       // Construct value from $token and $contact
 367
 368       $value = NULL;
 369
 370       if ($cfID = CRM_Core_BAO_CustomField::getKeyID($token)) {
 371         // only generate cv if we need it
 372         if ($cv === NULL) {
 373           $cv = CRM_Core_BAO_CustomValue::getContactValues($org['contact_id']);
 374         }
 375         foreach ($cv as $cvFieldID => $value) {
 376           if ($cvFieldID == $cfID) {
 377             $value = CRM_Core_BAO_CustomField::displayValue($value, $cfID);
 378             break;
 379           }
 380         }
 381       }
 382       elseif ($token == 'checksum') {
 383         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 384         $value = "cs={$cs}";
 385       }
 386       elseif ($token == 'address') {
 387         // Build the location values array
 388
 389         $loc = array();
 390         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 391         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 392         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 393         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 394         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 395
 396         // Construct the address token
 397
 398         $value = CRM_Utils_Address::format($loc);
 399         if ($html) {
 400           $value = str_replace("\n", '<br />', $value);
 401         }
 402       }
 403       else {
 404         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 405       }
 406
 407       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 408     }
 409
 410     return $str;
 411   }
 412
 413   /**
 414    * Replace all mailing tokens in $str
 415    *
 416    * @param string $str
 417    *   The string with tokens to be replaced.
 418    * @param object $mailing
 419    *   The mailing BAO, or null for validation.
 420    * @param bool $html
 421    *   Replace tokens with HTML or plain text.
 422    *
 423    * @param null $knownTokens
 424    * @param bool $escapeSmarty
 425    *
 426    * @return string
 427    *   The processed string
 428    */
 429   public static function &replaceMailingTokens(
 430     $str,
 431     &$mailing,
 432     $html = FALSE,
 433     $knownTokens = NULL,
 434     $escapeSmarty = FALSE
 435   ) {
 436     $key = 'mailing';
 437     if (!$knownTokens || !isset($knownTokens[$key])) {
 438       return $str;
 439     }
 440
 441     $str = preg_replace_callback(
 442       self::tokenRegex($key),
 443       function ($matches) use (&$mailing, $escapeSmarty) {
 444         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 445       },
 446       $str
 447     );
 448     return $str;
 449   }
 450
 451   /**
 452    * @param $token
 453    * @param $mailing
 454    * @param bool $escapeSmarty
 455    *
 456    * @return string
 457    */
 458   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 459     $value = '';
 460     switch ($token) {
 461       // CRM-7663
 462
 463       case 'id':
 464         $value = $mailing ? $mailing->id : 'undefined';
 465         break;
 466
 467       case 'name':
 468         $value = $mailing ? $mailing->name : 'Mailing Name';
 469         break;
 470
 471       case 'group':
 472         $groups = $mailing ? $mailing->getGroupNames() : array('Mailing Groups');
 473         $value = implode(', ', $groups);
 474         break;
 475
 476       case 'subject':
 477         $value = $mailing->subject;
 478         break;
 479
 480       case 'viewUrl':
 481         $mailingKey = $mailing->id;
 482         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 483           $mailingKey = $hash;
 484         }
 485         $value = CRM_Utils_System::url('civicrm/mailing/view',
 486           "reset=1&id={$mailingKey}",
 487           TRUE, NULL, FALSE, TRUE
 488         );
 489         break;
 490
 491       case 'editUrl':
 492       case 'scheduleUrl':
 493         // Note: editUrl and scheduleUrl used to be different, but now there's
 494         // one screen which can adapt based on permissions (in workflow mode).
 495         $value = CRM_Utils_System::url('civicrm/mailing/send',
 496           "reset=1&mid={$mailing->id}&continue=true",
 497           TRUE, NULL, FALSE, TRUE
 498         );
 499         break;
 500
 501       case 'html':
 502         $page = new CRM_Mailing_Page_View();
 503         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 504         break;
 505
 506       case 'approvalStatus':
 507         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 508         break;
 509
 510       case 'approvalNote':
 511         $value = $mailing->approval_note;
 512         break;
 513
 514       case 'approveUrl':
 515         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 516           "reset=1&mid={$mailing->id}",
 517           TRUE, NULL, FALSE, TRUE
 518         );
 519         break;
 520
 521       case 'creator':
 522         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 523         break;
 524
 525       case 'creatorEmail':
 526         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 527         break;
 528
 529       default:
 530         $value = "{mailing.$token}";
 531         break;
 532     }
 533
 534     if ($escapeSmarty) {
 535       $value = self::tokenEscapeSmarty($value);
 536     }
 537     return $value;
 538   }
 539
 540   /**
 541    * Replace all action tokens in $str
 542    *
 543    * @param string $str
 544    *   The string with tokens to be replaced.
 545    * @param array $addresses
 546    *   Assoc. array of VERP event addresses.
 547    * @param array $urls
 548    *   Assoc. array of action URLs.
 549    * @param bool $html
 550    *   Replace tokens with HTML or plain text.
 551    * @param array $knownTokens
 552    *   A list of tokens that are known to exist in the email body.
 553    *
 554    * @param bool $escapeSmarty
 555    *
 556    * @return string
 557    *   The processed string
 558    */
 559   public static function &replaceActionTokens(
 560     $str,
 561     &$addresses,
 562     &$urls,
 563     $html = FALSE,
 564     $knownTokens = NULL,
 565     $escapeSmarty = FALSE
 566   ) {
 567     $key = 'action';
 568     // here we intersect with the list of pre-configured valid tokens
 569     // so that we remove anything we do not recognize
 570     // I hope to move this step out of here soon and
 571     // then we will just iterate on a list of tokens that are passed to us
 572     if (!$knownTokens || empty($knownTokens[$key])) {
 573       return $str;
 574     }
 575
 576     $str = preg_replace_callback(
 577       self::tokenRegex($key),
 578       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 579         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 580       },
 581       $str
 582     );
 583     return $str;
 584   }
 585
 586   /**
 587    * @param $token
 588    * @param $addresses
 589    * @param $urls
 590    * @param bool $html
 591    * @param bool $escapeSmarty
 592    *
 593    * @return mixed|string
 594    */
 595   public static function getActionTokenReplacement(
 596     $token,
 597     &$addresses,
 598     &$urls,
 599     $html = FALSE,
 600     $escapeSmarty = FALSE
 601   ) {
 602     // If the token is an email action, use it.  Otherwise, find the
 603     // appropriate URL.
 604
 605     if (!in_array($token, self::$_tokens['action'])) {
 606       $value = "{action.$token}";
 607     }
 608     else {
 609       $value = CRM_Utils_Array::value($token, $addresses);
 610
 611       if ($value == NULL) {
 612         $value = CRM_Utils_Array::value($token, $urls);
 613       }
 614
 615       if ($value && $html) {
 616         // fix for CRM-2318
 617         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 618           $value = "mailto:$value";
 619         }
 620       }
 621       elseif ($value && !$html) {
 622         $value = str_replace('&amp;', '&', $value);
 623       }
 624     }
 625
 626     if ($escapeSmarty) {
 627       $value = self::tokenEscapeSmarty($value);
 628     }
 629     return $value;
 630   }
 631
 632   /**
 633    * Replace all the contact-level tokens in $str with information from
 634    * $contact.
 635    *
 636    * @param string $str
 637    *   The string with tokens to be replaced.
 638    * @param array $contact
 639    *   Associative array of contact properties.
 640    * @param bool $html
 641    *   Replace tokens with HTML or plain text.
 642    * @param array $knownTokens
 643    *   A list of tokens that are known to exist in the email body.
 644    * @param bool $returnBlankToken
 645    *   Return unevaluated token if value is null.
 646    *
 647    * @param bool $escapeSmarty
 648    *
 649    * @return string
 650    *   The processed string
 651    */
 652   public static function &replaceContactTokens(
 653     $str,
 654     &$contact,
 655     $html = FALSE,
 656     $knownTokens = NULL,
 657     $returnBlankToken = FALSE,
 658     $escapeSmarty = FALSE
 659   ) {
 660     $key = 'contact';
 661     if (self::$_tokens[$key] == NULL) {
 662       // This should come from UF
 663
 664       self::$_tokens[$key]
 665         = array_merge(
 666           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 667           array('checksum', 'contact_id')
 668         );
 669     }
 670
 671     // here we intersect with the list of pre-configured valid tokens
 672     // so that we remove anything we do not recognize
 673     // I hope to move this step out of here soon and
 674     // then we will just iterate on a list of tokens that are passed to us
 675     if (!$knownTokens || empty($knownTokens[$key])) {
 676       return $str;
 677     }
 678
 679     $str = preg_replace_callback(
 680       self::tokenRegex($key),
 681       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 682         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 683       },
 684       $str
 685     );
 686
 687     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 688     return $str;
 689   }
 690
 691   /**
 692    * @param $token
 693    * @param $contact
 694    * @param bool $html
 695    * @param bool $returnBlankToken
 696    * @param bool $escapeSmarty
 697    *
 698    * @return bool|mixed|null|string
 699    */
 700   public static function getContactTokenReplacement(
 701     $token,
 702     &$contact,
 703     $html = FALSE,
 704     $returnBlankToken = FALSE,
 705     $escapeSmarty = FALSE
 706   ) {
 707     if (self::$_tokens['contact'] == NULL) {
 708       /* This should come from UF */
 709
 710       self::$_tokens['contact']
 711         = array_merge(
 712           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 713           array('checksum', 'contact_id')
 714         );
 715     }
 716
 717     // Construct value from $token and $contact
 718
 719     $value = NULL;
 720     $noReplace = FALSE;
 721
 722     // Support legacy tokens
 723     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 724
 725     // check if the token we were passed is valid
 726     // we have to do this because this function is
 727     // called only when we find a token in the string
 728
 729     if (!in_array($token, self::$_tokens['contact'])) {
 730       $noReplace = TRUE;
 731     }
 732     elseif ($token == 'checksum') {
 733       $hash = CRM_Utils_Array::value('hash', $contact);
 734       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 735       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 736         NULL,
 737         NULL,
 738         $hash
 739       );
 740       $value = "cs={$cs}";
 741     }
 742     else {
 743       $value = CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 744
 745       // FIXME: for some pseudoconstants we get array ( 0 => id, 1 => label )
 746       if (is_array($value)) {
 747         $value = $value[1];
 748       }
 749       // Convert pseudoconstants using metadata
 750       elseif ($value && is_numeric($value)) {
 751         $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
 752         if (!empty($allFields[$token]['pseudoconstant'])) {
 753           $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
 754         }
 755       }
 756     }
 757
 758     if (!$html) {
 759       $value = str_replace('&amp;', '&', $value);
 760     }
 761
 762     // if null then return actual token
 763     if ($returnBlankToken && !$value) {
 764       $noReplace = TRUE;
 765     }
 766
 767     if ($noReplace) {
 768       $value = "{contact.$token}";
 769     }
 770
 771     if ($escapeSmarty
 772       && !($returnBlankToken && $noReplace)
 773     ) {
 774       // $returnBlankToken means the caller wants to do further attempts at
 775       // processing unreplaced tokens -- so don't escape them yet in this case.
 776       $value = self::tokenEscapeSmarty($value);
 777     }
 778
 779     return $value;
 780   }
 781
 782   /**
 783    * Replace all the hook tokens in $str with information from
 784    * $contact.
 785    *
 786    * @param string $str
 787    *   The string with tokens to be replaced.
 788    * @param array $contact
 789    *   Associative array of contact properties (including hook token values).
 790    * @param $categories
 791    * @param bool $html
 792    *   Replace tokens with HTML or plain text.
 793    *
 794    * @param bool $escapeSmarty
 795    *
 796    * @return string
 797    *   The processed string
 798    */
 799   public static function &replaceHookTokens(
 800     $str,
 801     &$contact,
 802     &$categories,
 803     $html = FALSE,
 804     $escapeSmarty = FALSE
 805   ) {
 806     foreach ($categories as $key) {
 807       $str = preg_replace_callback(
 808         self::tokenRegex($key),
 809         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 810           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 811         },
 812         $str
 813       );
 814     }
 815     return $str;
 816   }
 817
 818   /**
 819    * Parse html through Smarty resolving any smarty functions.
 820    * @param string $tokenHtml
 821    * @param array $entity
 822    * @param string $entityType
 823    * @return string
 824    *   html parsed through smarty
 825    */
 826   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 827     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 828       $smarty = CRM_Core_Smarty::singleton();
 829       // also add the tokens to the template
 830       $smarty->assign_by_ref($entityType, $entity);
 831       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 832     }
 833     return $tokenHtml;
 834   }
 835
 836   /**
 837    * @param $token
 838    * @param $contact
 839    * @param $category
 840    * @param bool $html
 841    * @param bool $escapeSmarty
 842    *
 843    * @return mixed|string
 844    */
 845   public static function getHookTokenReplacement(
 846     $token,
 847     &$contact,
 848     $category,
 849     $html = FALSE,
 850     $escapeSmarty = FALSE
 851   ) {
 852     $value = CRM_Utils_Array::value("{$category}.{$token}", $contact);
 853
 854     if ($value && !$html) {
 855       $value = str_replace('&amp;', '&', $value);
 856     }
 857
 858     if ($escapeSmarty) {
 859       $value = self::tokenEscapeSmarty($value);
 860     }
 861
 862     return $value;
 863   }
 864
 865   /**
 866    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 867    * for example {{token}}  or \{token}  will result in {token} in the final email
 868    *
 869    * this routine will remove the extra backslashes and braces
 870    *
 871    * @param $str ref to the string that will be scanned and modified
 872    */
 873   public static function unescapeTokens(&$str) {
 874     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 875   }
 876
 877   /**
 878    * Replace unsubscribe tokens.
 879    *
 880    * @param string $str
 881    *   The string with tokens to be replaced.
 882    * @param object $domain
 883    *   The domain BAO.
 884    * @param array $groups
 885    *   The groups (if any) being unsubscribed.
 886    * @param bool $html
 887    *   Replace tokens with html or plain text.
 888    * @param int $contact_id
 889    *   The contact ID.
 890    * @param string $hash The security hash of the unsub event
 891    *
 892    * @return string
 893    *   The processed string
 894    */
 895   public static function &replaceUnsubscribeTokens(
 896     $str,
 897     &$domain,
 898     &$groups,
 899     $html,
 900     $contact_id,
 901     $hash
 902   ) {
 903     if (self::token_match('unsubscribe', 'group', $str)) {
 904       if (!empty($groups)) {
 905         $config = CRM_Core_Config::singleton();
 906         $base = CRM_Utils_System::baseURL();
 907
 908         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 909         $dao = new CRM_Contact_DAO_Group();
 910         $dao->find();
 911         while ($dao->fetch()) {
 912           if (substr($dao->visibility, 0, 6) == 'Public') {
 913             $visibleGroups[] = $dao->id;
 914           }
 915         }
 916         $value = implode(', ', $groups);
 917         self::token_replace('unsubscribe', 'group', $value, $str);
 918       }
 919     }
 920     return $str;
 921   }
 922
 923   /**
 924    * Replace resubscribe tokens.
 925    *
 926    * @param string $str
 927    *   The string with tokens to be replaced.
 928    * @param object $domain
 929    *   The domain BAO.
 930    * @param array $groups
 931    *   The groups (if any) being resubscribed.
 932    * @param bool $html
 933    *   Replace tokens with html or plain text.
 934    * @param int $contact_id
 935    *   The contact ID.
 936    * @param string $hash The security hash of the resub event
 937    *
 938    * @return string
 939    *   The processed string
 940    */
 941   public static function &replaceResubscribeTokens(
 942     $str, &$domain, &$groups, $html,
 943     $contact_id, $hash
 944   ) {
 945     if (self::token_match('resubscribe', 'group', $str)) {
 946       if (!empty($groups)) {
 947         $value = implode(', ', $groups);
 948         self::token_replace('resubscribe', 'group', $value, $str);
 949       }
 950     }
 951     return $str;
 952   }
 953
 954   /**
 955    * Replace subscription-confirmation-request tokens
 956    *
 957    * @param string $str
 958    *   The string with tokens to be replaced.
 959    * @param string $group
 960    *   The name of the group being subscribed.
 961    * @param $url
 962    * @param bool $html
 963    *   Replace tokens with html or plain text.
 964    *
 965    * @return string
 966    *   The processed string
 967    */
 968   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 969     if (self::token_match('subscribe', 'group', $str)) {
 970       self::token_replace('subscribe', 'group', $group, $str);
 971     }
 972     if (self::token_match('subscribe', 'url', $str)) {
 973       self::token_replace('subscribe', 'url', $url, $str);
 974     }
 975     return $str;
 976   }
 977
 978   /**
 979    * Replace subscription-invitation tokens
 980    *
 981    * @param string $str
 982    *   The string with tokens to be replaced.
 983    *
 984    * @return string
 985    *   The processed string
 986    */
 987   public static function &replaceSubscribeInviteTokens($str) {
 988     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 989       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 990         'reset=1',
 991         TRUE, NULL, FALSE, TRUE
 992       );
 993       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 994     }
 995
 996     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
 997       foreach ($matches as $key => $value) {
 998         $gid = substr($value, 21, -1);
 999         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
1000           "reset=1&gid={$gid}",
1001           TRUE, NULL, FALSE, TRUE
1002         );
1003         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
1004       }
1005     }
1006
1007     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
1008       foreach ($matches as $key => $value) {
1009         $gid = substr($value, 18, -1);
1010         $config = CRM_Core_Config::singleton();
1011         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
1012         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
1013         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
1014         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1015       }
1016     }
1017     return $str;
1018   }
1019
1020   /**
1021    * Replace welcome/confirmation tokens
1022    *
1023    * @param string $str
1024    *   The string with tokens to be replaced.
1025    * @param string $group
1026    *   The name of the group being subscribed.
1027    * @param bool $html
1028    *   Replace tokens with html or plain text.
1029    *
1030    * @return string
1031    *   The processed string
1032    */
1033   public static function &replaceWelcomeTokens($str, $group, $html) {
1034     if (self::token_match('welcome', 'group', $str)) {
1035       self::token_replace('welcome', 'group', $group, $str);
1036     }
1037     return $str;
1038   }
1039
1040   /**
1041    * Find unprocessed tokens (call this last)
1042    *
1043    * @param string $str
1044    *   The string to search.
1045    *
1046    * @return array
1047    *   Array of tokens that weren't replaced
1048    */
1049   public static function &unmatchedTokens(&$str) {
1050     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1051     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1052     return $match[1];
1053   }
1054
1055   /**
1056    * Find and replace tokens for each component.
1057    *
1058    * @param string $str
1059    *   The string to search.
1060    * @param array $contact
1061    *   Associative array of contact properties.
1062    * @param array $components
1063    *   A list of tokens that are known to exist in the email body.
1064    *
1065    * @param bool $escapeSmarty
1066    * @param bool $returnEmptyToken
1067    *
1068    * @return string
1069    *   The processed string
1070    */
1071   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1072     if (!is_array($components) || empty($contact)) {
1073       return $str;
1074     }
1075
1076     foreach ($components as $name => $tokens) {
1077       if (!is_array($tokens) || empty($tokens)) {
1078         continue;
1079       }
1080
1081       foreach ($tokens as $token) {
1082         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1083           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1084         }
1085         elseif (!$returnEmptyToken) {
1086           //replacing empty token
1087           self::token_replace($name, $token, "", $str, $escapeSmarty);
1088         }
1089       }
1090     }
1091     return $str;
1092   }
1093
1094   /**
1095    * Get array of string tokens.
1096    *
1097    * @param string $string
1098    *   The input string to parse for tokens.
1099    *
1100    * @return array
1101    *   array of tokens mentioned in field
1102    */
1103   public static function getTokens($string) {
1104     $matches = array();
1105     $tokens = array();
1106     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1107       $string,
1108       $matches,
1109       PREG_PATTERN_ORDER
1110     );
1111
1112     if ($matches[1]) {
1113       foreach ($matches[1] as $token) {
1114         list($type, $name) = preg_split('/\./', $token, 2);
1115         if ($name && $type) {
1116           if (!isset($tokens[$type])) {
1117             $tokens[$type] = array();
1118           }
1119           $tokens[$type][] = $name;
1120         }
1121       }
1122     }
1123     return $tokens;
1124   }
1125
1126   /**
1127    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1128    * and getTokens appears to be historical rather than intentional and should be reviewed
1129    * @param $string
1130    * @return array
1131    *   fields to pass in as return properties when populating token
1132    */
1133   public static function getReturnProperties(&$string) {
1134     $returnProperties = array();
1135     $matches = array();
1136     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1137       $string,
1138       $matches,
1139       PREG_PATTERN_ORDER
1140     );
1141     if ($matches[1]) {
1142       foreach ($matches[1] as $token) {
1143         list($type, $name) = preg_split('/\./', $token, 2);
1144         if ($name) {
1145           $returnProperties["{$name}"] = 1;
1146         }
1147       }
1148     }
1149
1150     return $returnProperties;
1151   }
1152
1153   /**
1154    * Gives required details of contacts in an indexed array format so we
1155    * can iterate in a nice loop and do token evaluation
1156    *
1157    * @param $contactIDs
1158    * @param array $returnProperties
1159    *   Of required properties.
1160    * @param bool $skipOnHold Don't return on_hold contact info also.
1161    *   Don't return on_hold contact info also.
1162    * @param bool $skipDeceased Don't return deceased contact info.
1163    *   Don't return deceased contact info.
1164    * @param array $extraParams
1165    *   Extra params.
1166    * @param array $tokens
1167    *   The list of tokens we've extracted from the content.
1168    * @param null $className
1169    * @param int $jobID
1170    *   The mailing list jobID - this is a legacy param.
1171    *
1172    * @return array
1173    */
1174   public static function getTokenDetails(
1175     $contactIDs,
1176     $returnProperties = NULL,
1177     $skipOnHold = TRUE,
1178     $skipDeceased = TRUE,
1179     $extraParams = NULL,
1180     $tokens = array(),
1181     $className = NULL,
1182     $jobID = NULL
1183   ) {
1184     if (empty($contactIDs)) {
1185       // putting a fatal here so we can track if/when this happens
1186       CRM_Core_Error::fatal();
1187     }
1188     // @todo this functions needs unit tests.
1189     $params = array();
1190     foreach ($contactIDs as $key => $contactID) {
1191       $params[] = array(
1192         CRM_Core_Form::CB_PREFIX . $contactID,
1193         '=',
1194         1,
1195         0,
1196         0,
1197       );
1198     }
1199
1200     // fix for CRM-2613
1201     if ($skipDeceased) {
1202       $params[] = array('is_deceased', '=', 0, 0, 0);
1203     }
1204
1205     //fix for CRM-3798
1206     if ($skipOnHold) {
1207       $params[] = array('on_hold', '=', 0, 0, 0);
1208     }
1209
1210     if ($extraParams) {
1211       $params = array_merge($params, $extraParams);
1212     }
1213
1214     // if return properties are not passed then get all return properties
1215     if (empty($returnProperties)) {
1216       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1217         array('display_name', 'checksum', 'contact_id')
1218       );
1219       foreach ($fields as $key => $val) {
1220         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1221         // with contacts and are expensive to resolve.
1222         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1223         // and upstream calls to this should populate return properties.
1224         $unavailableFields = array('group', 'tag');
1225         if (!in_array($val, $unavailableFields)) {
1226           $returnProperties[$val] = 1;
1227         }
1228       }
1229     }
1230
1231     $custom = array();
1232     foreach ($returnProperties as $name => $dontCare) {
1233       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1234       if ($cfID) {
1235         $custom[] = $cfID;
1236       }
1237     }
1238
1239     //get the total number of contacts to fetch from database.
1240     $numberofContacts = count($contactIDs);
1241     $query = new CRM_Contact_BAO_Query($params, $returnProperties);
1242
1243     $details = $query->apiQuery($params, $returnProperties, NULL, NULL, 0, $numberofContacts);
1244
1245     $contactDetails = &$details[0];
1246
1247     foreach ($contactIDs as $key => $contactID) {
1248       if (array_key_exists($contactID, $contactDetails)) {
1249         if (CRM_Utils_Array::value('preferred_communication_method', $returnProperties) == 1
1250           && array_key_exists('preferred_communication_method', $contactDetails[$contactID])
1251         ) {
1252           $pcm = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method');
1253
1254           // communication Preference
1255           $contactPcm = explode(CRM_Core_DAO::VALUE_SEPARATOR,
1256             $contactDetails[$contactID]['preferred_communication_method']
1257           );
1258           $result = array();
1259           foreach ($contactPcm as $key => $val) {
1260             if ($val) {
1261               $result[$val] = $pcm[$val];
1262             }
1263           }
1264           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $result);
1265         }
1266
1267         foreach ($custom as $cfID) {
1268           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1269             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1270           }
1271         }
1272
1273         // special case for greeting replacement
1274         foreach (array(
1275                    'email_greeting',
1276                    'postal_greeting',
1277                    'addressee',
1278                  ) as $val) {
1279           if (!empty($contactDetails[$contactID][$val])) {
1280             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1281           }
1282         }
1283       }
1284     }
1285
1286     // also call a hook and get token details
1287     CRM_Utils_Hook::tokenValues($details[0],
1288       $contactIDs,
1289       $jobID,
1290       $tokens,
1291       $className
1292     );
1293     return $details;
1294   }
1295
1296   /**
1297    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1298    * specific tokens to be rendered
1299    *
1300    * @param array $contactIDs
1301    *   This should always be array(0) or its not anonymous - left to keep signature same.
1302    *   as main fn
1303    * @param string $returnProperties
1304    * @param bool $skipOnHold
1305    * @param bool $skipDeceased
1306    * @param string $extraParams
1307    * @param array $tokens
1308    * @param string $className
1309    *   Sent as context to the hook.
1310    * @param string $jobID
1311    * @return array
1312    *   contactDetails with hooks swapped out
1313    */
1314   public function getAnonymousTokenDetails($contactIDs = array(
1315       0,
1316     ),
1317                                            $returnProperties = NULL,
1318                                            $skipOnHold = TRUE,
1319                                            $skipDeceased = TRUE,
1320                                            $extraParams = NULL,
1321                                            $tokens = array(),
1322                                            $className = NULL,
1323                                            $jobID = NULL) {
1324     $details = array(0 => array());
1325     // also call a hook and get token details
1326     CRM_Utils_Hook::tokenValues($details[0],
1327       $contactIDs,
1328       $jobID,
1329       $tokens,
1330       $className
1331     );
1332     return $details;
1333   }
1334
1335   /**
1336    * Gives required details of contribuion in an indexed array format so we
1337    * can iterate in a nice loop and do token evaluation
1338    *
1339    * @param array $contributionIDs
1340    * @param array $returnProperties
1341    *   Of required properties.
1342    * @param array $extraParams
1343    *   Extra params.
1344    * @param array $tokens
1345    *   The list of tokens we've extracted from the content.
1346    * @param string $className
1347    *
1348    * @return array
1349    */
1350   public static function getContributionTokenDetails(
1351     $contributionIDs,
1352     $returnProperties = NULL,
1353     $extraParams = NULL,
1354     $tokens = array(),
1355     $className = NULL
1356   ) {
1357     // @todo this function basically replicates calling
1358     // civicrm_api3('contribution', 'get', array('id' => array('IN' => array())
1359     if (empty($contributionIDs)) {
1360       // putting a fatal here so we can track if/when this happens
1361       CRM_Core_Error::fatal();
1362     }
1363
1364     $details = array();
1365
1366     // no apiQuery helper yet, so do a loop and find contribution by id
1367     foreach ($contributionIDs as $contributionID) {
1368
1369       $dao = new CRM_Contribute_DAO_Contribution();
1370       $dao->id = $contributionID;
1371
1372       if ($dao->find(TRUE)) {
1373
1374         $details[$dao->id] = array();
1375         CRM_Core_DAO::storeValues($dao, $details[$dao->id]);
1376
1377         // do the necessary transformation
1378         if (!empty($details[$dao->id]['payment_instrument_id'])) {
1379           $piId = $details[$dao->id]['payment_instrument_id'];
1380           $pis = CRM_Contribute_PseudoConstant::paymentInstrument();
1381           $details[$dao->id]['payment_instrument'] = $pis[$piId];
1382         }
1383         if (!empty($details[$dao->id]['campaign_id'])) {
1384           $campaignId = $details[$dao->id]['campaign_id'];
1385           $campaigns = CRM_Campaign_BAO_Campaign::getCampaigns($campaignId);
1386           $details[$dao->id]['campaign'] = $campaigns[$campaignId];
1387         }
1388
1389         if (!empty($details[$dao->id]['financial_type_id'])) {
1390           $financialtypeId = $details[$dao->id]['financial_type_id'];
1391           $ftis = CRM_Contribute_PseudoConstant::financialType();
1392           $details[$dao->id]['financial_type'] = $ftis[$financialtypeId];
1393         }
1394
1395         // @todo call a hook to get token contribution details
1396       }
1397     }
1398
1399     return $details;
1400   }
1401
1402   /**
1403    * Get Membership Token Details.
1404    * @param array $membershipIDs
1405    *   Array of membership IDS.
1406    */
1407   public static function getMembershipTokenDetails($membershipIDs) {
1408     $memberships = civicrm_api3('membership', 'get', array(
1409         'options' => array('limit' => 200000),
1410         'membership_id' => array('IN' => (array) $membershipIDs),
1411       ));
1412     return $memberships['values'];
1413   }
1414
1415   /**
1416    * Replace existing greeting tokens in message/subject.
1417    *
1418    * @param string $tokenString
1419    * @param array $contactDetails
1420    * @param int $contactId
1421    * @param string $className
1422    * @param bool $escapeSmarty
1423    */
1424   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1425
1426     if (!$contactDetails && !$contactId) {
1427       return;
1428     }
1429
1430     // check if there are any tokens
1431     $greetingTokens = self::getTokens($tokenString);
1432
1433     if (!empty($greetingTokens)) {
1434       // first use the existing contact object for token replacement
1435       if (!empty($contactDetails)) {
1436         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1437       }
1438
1439       // check if there are any unevaluated tokens
1440       $greetingTokens = self::getTokens($tokenString);
1441
1442       // $greetingTokens not empty, means there are few tokens which are not
1443       // evaluated, like custom data etc
1444       // so retrieve it from database
1445       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1446         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1447         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1448         $contactParams = array('contact_id' => $contactId);
1449
1450         $greetingDetails = self::getTokenDetails($contactParams,
1451           $greetingsReturnProperties,
1452           FALSE, FALSE, NULL,
1453           $greetingTokens,
1454           $className
1455         );
1456
1457         // again replace tokens
1458         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1459           $greetingDetails,
1460           TRUE,
1461           $greetingTokens,
1462           TRUE,
1463           $escapeSmarty
1464         );
1465       }
1466
1467       // check if there are still any unevaluated tokens
1468       $remainingTokens = self::getTokens($tokenString);
1469
1470       // $greetingTokens not empty, there are customized or hook tokens to replace
1471       if (!empty($remainingTokens)) {
1472         // Fill the return properties array
1473         $greetingTokens = $remainingTokens;
1474         reset($greetingTokens);
1475         $greetingsReturnProperties = array();
1476         while (list($key) = each($greetingTokens)) {
1477           $props = array_flip(CRM_Utils_Array::value($key, $greetingTokens));
1478           $props = array_fill_keys(array_keys($props), 1);
1479           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1480         }
1481         $contactParams = array('contact_id' => $contactId);
1482         $greetingDetails = self::getTokenDetails($contactParams,
1483           $greetingsReturnProperties,
1484           FALSE, FALSE, NULL,
1485           $greetingTokens,
1486           $className
1487         );
1488         // Prepare variables for calling replaceHookTokens
1489         $categories = array_keys($greetingTokens);
1490         list($contact) = $greetingDetails;
1491         // Replace tokens defined in Hooks.
1492         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1493       }
1494     }
1495   }
1496
1497   /**
1498    * @param $tokens
1499    *
1500    * @return array
1501    */
1502   public static function flattenTokens(&$tokens) {
1503     $flattenTokens = array();
1504
1505     foreach (array(
1506                'html',
1507                'text',
1508                'subject',
1509              ) as $prop) {
1510       if (!isset($tokens[$prop])) {
1511         continue;
1512       }
1513       foreach ($tokens[$prop] as $type => $names) {
1514         if (!isset($flattenTokens[$type])) {
1515           $flattenTokens[$type] = array();
1516         }
1517         foreach ($names as $name) {
1518           $flattenTokens[$type][$name] = 1;
1519         }
1520       }
1521     }
1522
1523     return $flattenTokens;
1524   }
1525
1526   /**
1527    * Replace all user tokens in $str
1528    *
1529    * @param string $str
1530    *   The string with tokens to be replaced.
1531    *
1532    * @param null $knownTokens
1533    * @param bool $escapeSmarty
1534    *
1535    * @return string
1536    *   The processed string
1537    */
1538   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1539     $key = 'user';
1540     if (!$knownTokens ||
1541       !isset($knownTokens[$key])
1542     ) {
1543       return $str;
1544     }
1545
1546     $str = preg_replace_callback(
1547       self::tokenRegex($key),
1548       function ($matches) use ($escapeSmarty) {
1549         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1550       },
1551       $str
1552     );
1553     return $str;
1554   }
1555
1556   /**
1557    * @param $token
1558    * @param bool $escapeSmarty
1559    *
1560    * @return string
1561    */
1562   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1563     $value = '';
1564
1565     list($objectName, $objectValue) = explode('-', $token, 2);
1566
1567     switch ($objectName) {
1568       case 'permission':
1569         $value = CRM_Core_Permission::permissionEmails($objectValue);
1570         break;
1571
1572       case 'role':
1573         $value = CRM_Core_Permission::roleEmails($objectValue);
1574         break;
1575     }
1576
1577     if ($escapeSmarty) {
1578       $value = self::tokenEscapeSmarty($value);
1579     }
1580
1581     return $value;
1582   }
1583
1584   protected static function _buildContributionTokens() {
1585     $key = 'contribution';
1586     if (self::$_tokens[$key] == NULL) {
1587       self::$_tokens[$key] = array_keys(array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1588         array('campaign', 'financial_type')
1589       ));
1590     }
1591   }
1592
1593   /**
1594    * Store membership tokens on the static _tokens array.
1595    */
1596   protected static function _buildMembershipTokens() {
1597     $key = 'membership';
1598     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1599       $membershipTokens = array();
1600       $tokens = CRM_Core_SelectValues::membershipTokens();
1601       foreach ($tokens as $token => $dontCare) {
1602         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1603       }
1604       self::$_tokens[$key] = $membershipTokens;
1605     }
1606   }
1607
1608   /**
1609    * Replace tokens for an entity.
1610    * @param string $entity
1611    * @param array $entityArray
1612    *   (e.g. in format from api).
1613    * @param string $str
1614    *   String to replace in.
1615    * @param array $knownTokens
1616    *   Array of tokens present.
1617    * @param bool $escapeSmarty
1618    * @return string
1619    *   string with replacements made
1620    */
1621   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1622     if (!$knownTokens || empty($knownTokens[$entity])) {
1623       return $str;
1624     }
1625
1626     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1627     $fn = is_callable(array('CRM_Utils_Token', $fn)) ? $fn : 'getApiTokenReplacement';
1628     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1629     foreach ($knownTokens[$entity] as $token) {
1630       $replacement = self::$fn($entity, $token, $entityArray);
1631       if ($escapeSmarty) {
1632         $replacement = self::tokenEscapeSmarty($replacement);
1633       }
1634       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1635     }
1636     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1637   }
1638
1639   /**
1640    * @param int $caseId
1641    * @param int $str
1642    * @param array $knownTokens
1643    * @param bool $escapeSmarty
1644    * @return string
1645    * @throws \CiviCRM_API3_Exception
1646    */
1647   public static function replaceCaseTokens($caseId, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1648     if (!$knownTokens || empty($knownTokens['case'])) {
1649       return $str;
1650     }
1651     $case = civicrm_api3('case', 'getsingle', array('id' => $caseId));
1652     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1653   }
1654
1655   /**
1656    * Generic function for formatting token replacement for an api field
1657    *
1658    * @param string $entity
1659    * @param string $token
1660    * @param array $entityArray
1661    * @return string
1662    * @throws \CiviCRM_API3_Exception
1663    */
1664   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1665     if (!isset($entityArray[$token])) {
1666       return '';
1667     }
1668     $field = civicrm_api3($entity, 'getfield', array('action' => 'get', 'name' => $token, 'get_options' => 'get'));
1669     $field = $field['values'];
1670     $fieldType = CRM_Utils_Array::value('type', $field);
1671     // Boolean fields
1672     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1673       $field['options'] = array(ts('No'), ts('Yes'));
1674     }
1675     // Match pseudoconstants
1676     if (!empty($field['options'])) {
1677       $ret = array();
1678       foreach ((array) $entityArray[$token] as $val) {
1679         $ret[] = $field['options'][$val];
1680       }
1681       return implode(', ', $ret);
1682     }
1683     // Format date fields
1684     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1685       return CRM_Utils_Date::customFormat($entityArray[$token]);
1686     }
1687     return implode(', ', (array) $entityArray[$token]);
1688   }
1689
1690   /**
1691    * Replace Contribution tokens in html.
1692    *
1693    * @param string $str
1694    * @param array $contribution
1695    * @param bool|string $html
1696    * @param string $knownTokens
1697    * @param bool|string $escapeSmarty
1698    *
1699    * @return mixed
1700    */
1701   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1702     $key = 'contribution';
1703     if (!$knownTokens || !CRM_Utils_Array::value($key, $knownTokens)) {
1704       return $str; //early return
1705     }
1706     self::_buildContributionTokens();
1707
1708     // here we intersect with the list of pre-configured valid tokens
1709     // so that we remove anything we do not recognize
1710     // I hope to move this step out of here soon and
1711     // then we will just iterate on a list of tokens that are passed to us
1712
1713     $str = preg_replace_callback(
1714       self::tokenRegex($key),
1715       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1716         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1717       },
1718       $str
1719     );
1720
1721     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1722     return $str;
1723   }
1724
1725   /**
1726    * We have a situation where we are rendering more than one token in each field because we are combining
1727    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1728    *
1729    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1730    * it applies (other tokens are OK including date fields)
1731    *
1732    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1733    * does - but the fields is already being formatted through a date function
1734    *
1735    * @param string $separator
1736    * @param string $str
1737    * @param array $contribution
1738    * @param bool|string $html
1739    * @param string $knownTokens
1740    * @param bool|string $escapeSmarty
1741    *
1742    * @return string
1743    */
1744   public static function replaceMultipleContributionTokens($separator, $str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1745     if (empty($knownTokens['contribution'])) {
1746       return $str;
1747     }
1748
1749     if (in_array('receive_date', $knownTokens['contribution'])) {
1750       $formattedDates = array();
1751       $dates = explode($separator, $contribution['receive_date']);
1752       foreach ($dates as $date) {
1753         $formattedDates[] = CRM_Utils_Date::customFormat($date, NULL, array('j', 'm', 'Y'));
1754       }
1755       $str = str_replace("{contribution.receive_date}", implode($separator, $formattedDates), $str);
1756       unset($knownTokens['contribution']['receive_date']);
1757     }
1758     return self::replaceContributionTokens($str, $contribution, $html, $knownTokens, $escapeSmarty);
1759   }
1760
1761   /**
1762    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1763    * - this is used by the pdfLetter task from membership search
1764    * @param string $entity
1765    *   should always be "membership"
1766    * @param string $token
1767    *   field name
1768    * @param array $membership
1769    *   An api result array for a single membership.
1770    * @return string token replacement
1771    */
1772   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1773     self::_buildMembershipTokens();
1774     switch ($token) {
1775       case 'type':
1776         $value = $membership['membership_name'];
1777         break;
1778
1779       case 'status':
1780         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1781         $value = $statuses[$membership['status_id']];
1782         break;
1783
1784       case 'fee':
1785         try {
1786           $value = civicrm_api3('membership_type', 'getvalue', array(
1787               'id' => $membership['membership_type_id'],
1788               'return' => 'minimum_fee',
1789             ));
1790         }
1791         catch (CiviCRM_API3_Exception $e) {
1792           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1793           // api handles NULL (4.4)
1794           $value = 0;
1795         }
1796         break;
1797
1798       default:
1799         if (in_array($token, self::$_tokens[$entity])) {
1800           $value = $membership[$token];
1801         }
1802         else {
1803           // ie unchanged
1804           $value = "{$entity}.{$token}";
1805         }
1806         break;
1807     }
1808
1809     return $value;
1810   }
1811
1812   /**
1813    * @param $token
1814    * @param $contribution
1815    * @param bool $html
1816    * @param bool $escapeSmarty
1817    *
1818    * @return mixed|string
1819    */
1820   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1821     self::_buildContributionTokens();
1822
1823     switch ($token) {
1824       case 'total_amount':
1825       case 'net_amount':
1826       case 'fee_amount':
1827       case 'non_deductible_amount':
1828         $value = CRM_Utils_Money::format(CRM_Utils_Array::retrieveValueRecursive($contribution, $token));
1829         break;
1830
1831       case 'receive_date':
1832         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1833         $value = CRM_Utils_Date::customFormat($value, NULL, array('j', 'm', 'Y'));
1834         break;
1835
1836       default:
1837         if (!in_array($token, self::$_tokens['contribution'])) {
1838           $value = "{contribution.$token}";
1839         }
1840         else {
1841           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1842         }
1843         break;
1844     }
1845
1846     if ($escapeSmarty) {
1847       $value = self::tokenEscapeSmarty($value);
1848     }
1849     return $value;
1850   }
1851
1852   /**
1853    * @return array
1854    *   [legacy_token => new_token]
1855    */
1856   public static function legacyContactTokens() {
1857     return array(
1858       'individual_prefix' => 'prefix_id',
1859       'individual_suffix' => 'suffix_id',
1860       'gender' => 'gender_id',
1861       'communication_style' => 'communication_style_id',
1862     );
1863   }
1864
1865   /**
1866    * Formats a token list for the select2 widget
1867    *
1868    * @param $tokens
1869    * @return array
1870    */
1871   public static function formatTokensForDisplay($tokens) {
1872     $sorted = $output = array();
1873
1874     // Sort in ascending order by ignoring word case
1875     natcasesort($tokens);
1876
1877     // Attempt to place tokens into optgroups
1878     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1879     foreach ($tokens as $k => $v) {
1880       // Check to see if this token is already in a group e.g. for custom fields
1881       $split = explode(' :: ', $v);
1882       if (!empty($split[1])) {
1883         $sorted[$split[1]][] = array('id' => $k, 'text' => $split[0]);
1884       }
1885       // Group by entity
1886       else {
1887         $split = explode('.', trim($k, '{}'));
1888         if (isset($split[1])) {
1889           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucfirst($split[0]);
1890         }
1891         else {
1892           $entity = 'Contact';
1893         }
1894         $sorted[ts($entity)][] = array('id' => $k, 'text' => $v);
1895       }
1896     }
1897
1898     ksort($sorted);
1899     foreach ($sorted as $k => $v) {
1900       $output[] = array('text' => $k, 'children' => $v);
1901     }
1902
1903     return $output;
1904   }
1905
1906 }