CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2017                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2017
  32  */
  33
  34 /**
  35  * Class to abstract token replacement.
  36  */
  37 class CRM_Utils_Token {
  38   static $_requiredTokens = NULL;
  39
  40   static $_tokens = array(
  41     'action' => array(
  42       'forward',
  43       'optOut',
  44       'optOutUrl',
  45       'reply',
  46       'unsubscribe',
  47       'unsubscribeUrl',
  48       'resubscribe',
  49       'resubscribeUrl',
  50       'subscribeUrl',
  51     ),
  52     'mailing' => array(
  53       'id',
  54       'name',
  55       'group',
  56       'subject',
  57       'viewUrl',
  58       'editUrl',
  59       'scheduleUrl',
  60       'approvalStatus',
  61       'approvalNote',
  62       'approveUrl',
  63       'creator',
  64       'creatorEmail',
  65     ),
  66     'user' => array(
  67       // we extract the stuff after the role / permission and return the
  68       // civicrm email addresses of all users with that role / permission
  69       // useful with rules integration
  70       'permission:',
  71       'role:',
  72     ),
  73     // populate this dynamically
  74     'contact' => NULL,
  75     // populate this dynamically
  76     'contribution' => NULL,
  77     'domain' => array(
  78       'name',
  79       'phone',
  80       'address',
  81       'email',
  82       'id',
  83       'description',
  84     ),
  85     'subscribe' => array('group'),
  86     'unsubscribe' => array('group'),
  87     'resubscribe' => array('group'),
  88     'welcome' => array('group'),
  89   );
  90
  91
  92   /**
  93    * @deprecated
  94    *   This is used by CiviMail but will be made redundant by FlexMailer.
  95    * @return array
  96    */
  97   public static function getRequiredTokens() {
  98     if (self::$_requiredTokens == NULL) {
  99       self::$_requiredTokens = array(
 100         'domain.address' => ts("Domain address - displays your organization's postal address."),
 101         'action.optOutUrl or action.unsubscribeUrl' => array(
 102           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
 103           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
 104           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
 105           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
 106         ),
 107       );
 108     }
 109     return self::$_requiredTokens;
 110   }
 111
 112   /**
 113    * Check a string (mailing body) for required tokens.
 114    *
 115    * @param string $str
 116    *   The message.
 117    *
 118    * @return bool|array
 119    *    true if all required tokens are found,
 120    *    else an array of the missing tokens
 121    */
 122   public static function requiredTokens(&$str) {
 123     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 124     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, array()) : CRM_Utils_Token::getRequiredTokens();
 125
 126     $missing = array();
 127     foreach ($requiredTokens as $token => $value) {
 128       if (!is_array($value)) {
 129         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 130           $missing[$token] = $value;
 131         }
 132       }
 133       else {
 134         $present = FALSE;
 135         $desc = NULL;
 136         foreach ($value as $t => $d) {
 137           $desc = $d;
 138           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 139             $present = TRUE;
 140           }
 141         }
 142         if (!$present) {
 143           $missing[$token] = $desc;
 144         }
 145       }
 146     }
 147
 148     if (empty($missing)) {
 149       return TRUE;
 150     }
 151     return $missing;
 152   }
 153
 154   /**
 155    * Wrapper for token matching.
 156    *
 157    * @param string $type
 158    *   The token type (domain,mailing,contact,action).
 159    * @param string $var
 160    *   The token variable.
 161    * @param string $str
 162    *   The string to search.
 163    *
 164    * @return bool
 165    *   Was there a match
 166    */
 167   public static function token_match($type, $var, &$str) {
 168     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 169     return preg_match("/(^|[^\{])$token/", $str);
 170   }
 171
 172   /**
 173    * Wrapper for token replacing.
 174    *
 175    * @param string $type
 176    *   The token type.
 177    * @param string $var
 178    *   The token variable.
 179    * @param string $value
 180    *   The value to substitute for the token.
 181    * @param string (reference) $str The string to replace in
 182    *
 183    * @param bool $escapeSmarty
 184    *
 185    * @return string
 186    *   The processed string
 187    */
 188   public static function &token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 189     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 190     if (!$value) {
 191       $value = '$3';
 192     }
 193     if ($escapeSmarty) {
 194       $value = self::tokenEscapeSmarty($value);
 195     }
 196     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 197     return $str;
 198   }
 199
 200   /**
 201    * Get< the regex for token replacement
 202    *
 203    * @param string $token_type
 204    *   A string indicating the the type of token to be used in the expression.
 205    *
 206    * @return string
 207    *   regular expression sutiable for using in preg_replace
 208    */
 209   private static function tokenRegex($token_type) {
 210     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(\-[\w\s]+)?)\}(?!\})/';
 211   }
 212
 213   /**
 214    * Escape the string so a malicious user cannot inject smarty code into the template.
 215    *
 216    * @param string $string
 217    *   A string that needs to be escaped from smarty parsing.
 218    *
 219    * @return string
 220    *   the escaped string
 221    */
 222   public static function tokenEscapeSmarty($string) {
 223     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 224     return preg_replace(array('/{/', '/(?<!{ldelim)}/'), array('{ldelim}', '{rdelim}'), $string);
 225   }
 226
 227   /**
 228    * Replace all the domain-level tokens in $str
 229    *
 230    * @param string $str
 231    *   The string with tokens to be replaced.
 232    * @param object $domain
 233    *   The domain BAO.
 234    * @param bool $html
 235    *   Replace tokens with HTML or plain text.
 236    *
 237    * @param null $knownTokens
 238    * @param bool $escapeSmarty
 239    *
 240    * @return string
 241    *   The processed string
 242    */
 243   public static function &replaceDomainTokens(
 244     $str,
 245     &$domain,
 246     $html = FALSE,
 247     $knownTokens = NULL,
 248     $escapeSmarty = FALSE
 249   ) {
 250     $key = 'domain';
 251     if (
 252       !$knownTokens || empty($knownTokens[$key])
 253     ) {
 254       return $str;
 255     }
 256
 257     $str = preg_replace_callback(
 258       self::tokenRegex($key),
 259       function ($matches) use (&$domain, $html, $escapeSmarty) {
 260         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 261       },
 262       $str
 263     );
 264     return $str;
 265   }
 266
 267   /**
 268    * @param $token
 269    * @param $domain
 270    * @param bool $html
 271    * @param bool $escapeSmarty
 272    *
 273    * @return mixed|null|string
 274    */
 275   public static function getDomainTokenReplacement($token, &$domain, $html = FALSE, $escapeSmarty = FALSE) {
 276     // check if the token we were passed is valid
 277     // we have to do this because this function is
 278     // called only when we find a token in the string
 279
 280     $loc = &$domain->getLocationValues();
 281
 282     if (!in_array($token, self::$_tokens['domain'])) {
 283       $value = "{domain.$token}";
 284     }
 285     elseif ($token == 'address') {
 286       static $addressCache = array();
 287
 288       $cache_key = $html ? 'address-html' : 'address-text';
 289       if (array_key_exists($cache_key, $addressCache)) {
 290         return $addressCache[$cache_key];
 291       }
 292
 293       $value = NULL;
 294       // Construct the address token
 295
 296       if (!empty($loc[$token])) {
 297         if ($html) {
 298           $value = $loc[$token][1]['display'];
 299           $value = str_replace("\n", '<br />', $value);
 300         }
 301         else {
 302           $value = $loc[$token][1]['display_text'];
 303         }
 304         $addressCache[$cache_key] = $value;
 305       }
 306     }
 307     elseif ($token == 'name' || $token == 'id' || $token == 'description') {
 308       $value = $domain->$token;
 309     }
 310     elseif ($token == 'phone' || $token == 'email') {
 311       // Construct the phone and email tokens
 312
 313       $value = NULL;
 314       if (!empty($loc[$token])) {
 315         foreach ($loc[$token] as $index => $entity) {
 316           $value = $entity[$token];
 317           break;
 318         }
 319       }
 320     }
 321
 322     if ($escapeSmarty) {
 323       $value = self::tokenEscapeSmarty($value);
 324     }
 325
 326     return $value;
 327   }
 328
 329   /**
 330    * Replace all the org-level tokens in $str
 331    *
 332    * @fixme: This function appears to be broken, as it depends on
 333    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 334    * Marking as deprecated until this is fixed
 335    * @deprecated
 336    *
 337    * @param string $str
 338    *   The string with tokens to be replaced.
 339    * @param object $org
 340    *   Associative array of org properties.
 341    * @param bool $html
 342    *   Replace tokens with HTML or plain text.
 343    *
 344    * @param bool $escapeSmarty
 345    *
 346    * @return string
 347    *   The processed string
 348    */
 349   public static function &replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 350     self::$_tokens['org']
 351       = array_merge(
 352         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 353         array('address', 'display_name', 'checksum', 'contact_id')
 354       );
 355
 356     $cv = NULL;
 357     foreach (self::$_tokens['org'] as $token) {
 358       // print "Getting token value for $token<br/><br/>";
 359       if ($token == '') {
 360         continue;
 361       }
 362
 363       // If the string doesn't contain this token, skip it.
 364
 365       if (!self::token_match('org', $token, $str)) {
 366         continue;
 367       }
 368
 369       // Construct value from $token and $contact
 370
 371       $value = NULL;
 372
 373       if ($cfID = CRM_Core_BAO_CustomField::getKeyID($token)) {
 374         // only generate cv if we need it
 375         if ($cv === NULL) {
 376           $cv = CRM_Core_BAO_CustomValue::getContactValues($org['contact_id']);
 377         }
 378         foreach ($cv as $cvFieldID => $value) {
 379           if ($cvFieldID == $cfID) {
 380             $value = CRM_Core_BAO_CustomField::displayValue($value, $cfID);
 381             break;
 382           }
 383         }
 384       }
 385       elseif ($token == 'checksum') {
 386         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 387         $value = "cs={$cs}";
 388       }
 389       elseif ($token == 'address') {
 390         // Build the location values array
 391
 392         $loc = array();
 393         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 394         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 395         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 396         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 397         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 398
 399         // Construct the address token
 400
 401         $value = CRM_Utils_Address::format($loc);
 402         if ($html) {
 403           $value = str_replace("\n", '<br />', $value);
 404         }
 405       }
 406       else {
 407         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 408       }
 409
 410       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 411     }
 412
 413     return $str;
 414   }
 415
 416   /**
 417    * Replace all mailing tokens in $str
 418    *
 419    * @param string $str
 420    *   The string with tokens to be replaced.
 421    * @param object $mailing
 422    *   The mailing BAO, or null for validation.
 423    * @param bool $html
 424    *   Replace tokens with HTML or plain text.
 425    *
 426    * @param null $knownTokens
 427    * @param bool $escapeSmarty
 428    *
 429    * @return string
 430    *   The processed string
 431    */
 432   public static function &replaceMailingTokens(
 433     $str,
 434     &$mailing,
 435     $html = FALSE,
 436     $knownTokens = NULL,
 437     $escapeSmarty = FALSE
 438   ) {
 439     $key = 'mailing';
 440     if (!$knownTokens || !isset($knownTokens[$key])) {
 441       return $str;
 442     }
 443
 444     $str = preg_replace_callback(
 445       self::tokenRegex($key),
 446       function ($matches) use (&$mailing, $escapeSmarty) {
 447         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 448       },
 449       $str
 450     );
 451     return $str;
 452   }
 453
 454   /**
 455    * @param $token
 456    * @param $mailing
 457    * @param bool $escapeSmarty
 458    *
 459    * @return string
 460    */
 461   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 462     $value = '';
 463     switch ($token) {
 464       // CRM-7663
 465
 466       case 'id':
 467         $value = $mailing ? $mailing->id : 'undefined';
 468         break;
 469
 470       case 'name':
 471         $value = $mailing ? $mailing->name : 'Mailing Name';
 472         break;
 473
 474       case 'group':
 475         $groups = $mailing ? $mailing->getGroupNames() : array('Mailing Groups');
 476         $value = implode(', ', $groups);
 477         break;
 478
 479       case 'subject':
 480         $value = $mailing->subject;
 481         break;
 482
 483       case 'viewUrl':
 484         $mailingKey = $mailing->id;
 485         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 486           $mailingKey = $hash;
 487         }
 488         $value = CRM_Utils_System::url('civicrm/mailing/view',
 489           "reset=1&id={$mailingKey}",
 490           TRUE, NULL, FALSE, TRUE
 491         );
 492         break;
 493
 494       case 'editUrl':
 495       case 'scheduleUrl':
 496         // Note: editUrl and scheduleUrl used to be different, but now there's
 497         // one screen which can adapt based on permissions (in workflow mode).
 498         $value = CRM_Utils_System::url('civicrm/mailing/send',
 499           "reset=1&mid={$mailing->id}&continue=true",
 500           TRUE, NULL, FALSE, TRUE
 501         );
 502         break;
 503
 504       case 'html':
 505         $page = new CRM_Mailing_Page_View();
 506         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 507         break;
 508
 509       case 'approvalStatus':
 510         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 511         break;
 512
 513       case 'approvalNote':
 514         $value = $mailing->approval_note;
 515         break;
 516
 517       case 'approveUrl':
 518         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 519           "reset=1&mid={$mailing->id}",
 520           TRUE, NULL, FALSE, TRUE
 521         );
 522         break;
 523
 524       case 'creator':
 525         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 526         break;
 527
 528       case 'creatorEmail':
 529         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 530         break;
 531
 532       default:
 533         $value = "{mailing.$token}";
 534         break;
 535     }
 536
 537     if ($escapeSmarty) {
 538       $value = self::tokenEscapeSmarty($value);
 539     }
 540     return $value;
 541   }
 542
 543   /**
 544    * Replace all action tokens in $str
 545    *
 546    * @param string $str
 547    *   The string with tokens to be replaced.
 548    * @param array $addresses
 549    *   Assoc. array of VERP event addresses.
 550    * @param array $urls
 551    *   Assoc. array of action URLs.
 552    * @param bool $html
 553    *   Replace tokens with HTML or plain text.
 554    * @param array $knownTokens
 555    *   A list of tokens that are known to exist in the email body.
 556    *
 557    * @param bool $escapeSmarty
 558    *
 559    * @return string
 560    *   The processed string
 561    */
 562   public static function &replaceActionTokens(
 563     $str,
 564     &$addresses,
 565     &$urls,
 566     $html = FALSE,
 567     $knownTokens = NULL,
 568     $escapeSmarty = FALSE
 569   ) {
 570     $key = 'action';
 571     // here we intersect with the list of pre-configured valid tokens
 572     // so that we remove anything we do not recognize
 573     // I hope to move this step out of here soon and
 574     // then we will just iterate on a list of tokens that are passed to us
 575     if (!$knownTokens || empty($knownTokens[$key])) {
 576       return $str;
 577     }
 578
 579     $str = preg_replace_callback(
 580       self::tokenRegex($key),
 581       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 582         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 583       },
 584       $str
 585     );
 586     return $str;
 587   }
 588
 589   /**
 590    * @param $token
 591    * @param $addresses
 592    * @param $urls
 593    * @param bool $html
 594    * @param bool $escapeSmarty
 595    *
 596    * @return mixed|string
 597    */
 598   public static function getActionTokenReplacement(
 599     $token,
 600     &$addresses,
 601     &$urls,
 602     $html = FALSE,
 603     $escapeSmarty = FALSE
 604   ) {
 605     // If the token is an email action, use it.  Otherwise, find the
 606     // appropriate URL.
 607
 608     if (!in_array($token, self::$_tokens['action'])) {
 609       $value = "{action.$token}";
 610     }
 611     else {
 612       $value = CRM_Utils_Array::value($token, $addresses);
 613
 614       if ($value == NULL) {
 615         $value = CRM_Utils_Array::value($token, $urls);
 616       }
 617
 618       if ($value && $html) {
 619         // fix for CRM-2318
 620         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 621           $value = "mailto:$value";
 622         }
 623       }
 624       elseif ($value && !$html) {
 625         $value = str_replace('&amp;', '&', $value);
 626       }
 627     }
 628
 629     if ($escapeSmarty) {
 630       $value = self::tokenEscapeSmarty($value);
 631     }
 632     return $value;
 633   }
 634
 635   /**
 636    * Replace all the contact-level tokens in $str with information from
 637    * $contact.
 638    *
 639    * @param string $str
 640    *   The string with tokens to be replaced.
 641    * @param array $contact
 642    *   Associative array of contact properties.
 643    * @param bool $html
 644    *   Replace tokens with HTML or plain text.
 645    * @param array $knownTokens
 646    *   A list of tokens that are known to exist in the email body.
 647    * @param bool $returnBlankToken
 648    *   Return unevaluated token if value is null.
 649    *
 650    * @param bool $escapeSmarty
 651    *
 652    * @return string
 653    *   The processed string
 654    */
 655   public static function &replaceContactTokens(
 656     $str,
 657     &$contact,
 658     $html = FALSE,
 659     $knownTokens = NULL,
 660     $returnBlankToken = FALSE,
 661     $escapeSmarty = FALSE
 662   ) {
 663     $key = 'contact';
 664     if (self::$_tokens[$key] == NULL) {
 665       // This should come from UF
 666
 667       self::$_tokens[$key]
 668         = array_merge(
 669           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 670           array('checksum', 'contact_id')
 671         );
 672     }
 673
 674     // here we intersect with the list of pre-configured valid tokens
 675     // so that we remove anything we do not recognize
 676     // I hope to move this step out of here soon and
 677     // then we will just iterate on a list of tokens that are passed to us
 678     if (!$knownTokens || empty($knownTokens[$key])) {
 679       return $str;
 680     }
 681
 682     $str = preg_replace_callback(
 683       self::tokenRegex($key),
 684       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 685         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 686       },
 687       $str
 688     );
 689
 690     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 691     return $str;
 692   }
 693
 694   /**
 695    * @param $token
 696    * @param $contact
 697    * @param bool $html
 698    * @param bool $returnBlankToken
 699    * @param bool $escapeSmarty
 700    *
 701    * @return bool|mixed|null|string
 702    */
 703   public static function getContactTokenReplacement(
 704     $token,
 705     &$contact,
 706     $html = FALSE,
 707     $returnBlankToken = FALSE,
 708     $escapeSmarty = FALSE
 709   ) {
 710     if (self::$_tokens['contact'] == NULL) {
 711       /* This should come from UF */
 712
 713       self::$_tokens['contact']
 714         = array_merge(
 715           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 716           array('checksum', 'contact_id')
 717         );
 718     }
 719
 720     // Construct value from $token and $contact
 721
 722     $value = NULL;
 723     $noReplace = FALSE;
 724
 725     // Support legacy tokens
 726     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 727
 728     // check if the token we were passed is valid
 729     // we have to do this because this function is
 730     // called only when we find a token in the string
 731
 732     if (!in_array($token, self::$_tokens['contact'])) {
 733       $noReplace = TRUE;
 734     }
 735     elseif ($token == 'checksum') {
 736       $hash = CRM_Utils_Array::value('hash', $contact);
 737       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 738       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 739         NULL,
 740         NULL,
 741         $hash
 742       );
 743       $value = "cs={$cs}";
 744     }
 745     else {
 746       $value = CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 747
 748       // FIXME: for some pseudoconstants we get array ( 0 => id, 1 => label )
 749       if (is_array($value)) {
 750         $value = $value[1];
 751       }
 752       // Convert pseudoconstants using metadata
 753       elseif ($value && is_numeric($value)) {
 754         $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
 755         if (!empty($allFields[$token]['pseudoconstant'])) {
 756           $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
 757         }
 758       }
 759       elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
 760         $value = CRM_Utils_Date::customFormat($value);
 761       }
 762     }
 763
 764     if (!$html) {
 765       $value = str_replace('&amp;', '&', $value);
 766     }
 767
 768     // if null then return actual token
 769     if ($returnBlankToken && !$value) {
 770       $noReplace = TRUE;
 771     }
 772
 773     if ($noReplace) {
 774       $value = "{contact.$token}";
 775     }
 776
 777     if ($escapeSmarty
 778       && !($returnBlankToken && $noReplace)
 779     ) {
 780       // $returnBlankToken means the caller wants to do further attempts at
 781       // processing unreplaced tokens -- so don't escape them yet in this case.
 782       $value = self::tokenEscapeSmarty($value);
 783     }
 784
 785     return $value;
 786   }
 787
 788   /**
 789    * Replace all the hook tokens in $str with information from
 790    * $contact.
 791    *
 792    * @param string $str
 793    *   The string with tokens to be replaced.
 794    * @param array $contact
 795    *   Associative array of contact properties (including hook token values).
 796    * @param $categories
 797    * @param bool $html
 798    *   Replace tokens with HTML or plain text.
 799    *
 800    * @param bool $escapeSmarty
 801    *
 802    * @return string
 803    *   The processed string
 804    */
 805   public static function &replaceHookTokens(
 806     $str,
 807     &$contact,
 808     &$categories,
 809     $html = FALSE,
 810     $escapeSmarty = FALSE
 811   ) {
 812     foreach ($categories as $key) {
 813       $str = preg_replace_callback(
 814         self::tokenRegex($key),
 815         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 816           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 817         },
 818         $str
 819       );
 820     }
 821     return $str;
 822   }
 823
 824   /**
 825    * Parse html through Smarty resolving any smarty functions.
 826    * @param string $tokenHtml
 827    * @param array $entity
 828    * @param string $entityType
 829    * @return string
 830    *   html parsed through smarty
 831    */
 832   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 833     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 834       $smarty = CRM_Core_Smarty::singleton();
 835       // also add the tokens to the template
 836       $smarty->assign_by_ref($entityType, $entity);
 837       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 838     }
 839     return $tokenHtml;
 840   }
 841
 842   /**
 843    * @param $token
 844    * @param $contact
 845    * @param $category
 846    * @param bool $html
 847    * @param bool $escapeSmarty
 848    *
 849    * @return mixed|string
 850    */
 851   public static function getHookTokenReplacement(
 852     $token,
 853     &$contact,
 854     $category,
 855     $html = FALSE,
 856     $escapeSmarty = FALSE
 857   ) {
 858     $value = CRM_Utils_Array::value("{$category}.{$token}", $contact);
 859
 860     if ($value && !$html) {
 861       $value = str_replace('&amp;', '&', $value);
 862     }
 863
 864     if ($escapeSmarty) {
 865       $value = self::tokenEscapeSmarty($value);
 866     }
 867
 868     return $value;
 869   }
 870
 871   /**
 872    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 873    * for example {{token}}  or \{token}  will result in {token} in the final email
 874    *
 875    * this routine will remove the extra backslashes and braces
 876    *
 877    * @param $str ref to the string that will be scanned and modified
 878    */
 879   public static function unescapeTokens(&$str) {
 880     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 881   }
 882
 883   /**
 884    * Replace unsubscribe tokens.
 885    *
 886    * @param string $str
 887    *   The string with tokens to be replaced.
 888    * @param object $domain
 889    *   The domain BAO.
 890    * @param array $groups
 891    *   The groups (if any) being unsubscribed.
 892    * @param bool $html
 893    *   Replace tokens with html or plain text.
 894    * @param int $contact_id
 895    *   The contact ID.
 896    * @param string $hash The security hash of the unsub event
 897    *
 898    * @return string
 899    *   The processed string
 900    */
 901   public static function &replaceUnsubscribeTokens(
 902     $str,
 903     &$domain,
 904     &$groups,
 905     $html,
 906     $contact_id,
 907     $hash
 908   ) {
 909     if (self::token_match('unsubscribe', 'group', $str)) {
 910       if (!empty($groups)) {
 911         $config = CRM_Core_Config::singleton();
 912         $base = CRM_Utils_System::baseURL();
 913
 914         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 915         $dao = new CRM_Contact_DAO_Group();
 916         $dao->find();
 917         while ($dao->fetch()) {
 918           if (substr($dao->visibility, 0, 6) == 'Public') {
 919             $visibleGroups[] = $dao->id;
 920           }
 921         }
 922         $value = implode(', ', $groups);
 923         self::token_replace('unsubscribe', 'group', $value, $str);
 924       }
 925     }
 926     return $str;
 927   }
 928
 929   /**
 930    * Replace resubscribe tokens.
 931    *
 932    * @param string $str
 933    *   The string with tokens to be replaced.
 934    * @param object $domain
 935    *   The domain BAO.
 936    * @param array $groups
 937    *   The groups (if any) being resubscribed.
 938    * @param bool $html
 939    *   Replace tokens with html or plain text.
 940    * @param int $contact_id
 941    *   The contact ID.
 942    * @param string $hash The security hash of the resub event
 943    *
 944    * @return string
 945    *   The processed string
 946    */
 947   public static function &replaceResubscribeTokens(
 948     $str, &$domain, &$groups, $html,
 949     $contact_id, $hash
 950   ) {
 951     if (self::token_match('resubscribe', 'group', $str)) {
 952       if (!empty($groups)) {
 953         $value = implode(', ', $groups);
 954         self::token_replace('resubscribe', 'group', $value, $str);
 955       }
 956     }
 957     return $str;
 958   }
 959
 960   /**
 961    * Replace subscription-confirmation-request tokens
 962    *
 963    * @param string $str
 964    *   The string with tokens to be replaced.
 965    * @param string $group
 966    *   The name of the group being subscribed.
 967    * @param $url
 968    * @param bool $html
 969    *   Replace tokens with html or plain text.
 970    *
 971    * @return string
 972    *   The processed string
 973    */
 974   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 975     if (self::token_match('subscribe', 'group', $str)) {
 976       self::token_replace('subscribe', 'group', $group, $str);
 977     }
 978     if (self::token_match('subscribe', 'url', $str)) {
 979       self::token_replace('subscribe', 'url', $url, $str);
 980     }
 981     return $str;
 982   }
 983
 984   /**
 985    * Replace subscription-invitation tokens
 986    *
 987    * @param string $str
 988    *   The string with tokens to be replaced.
 989    *
 990    * @return string
 991    *   The processed string
 992    */
 993   public static function &replaceSubscribeInviteTokens($str) {
 994     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 995       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 996         'reset=1',
 997         TRUE, NULL, FALSE, TRUE
 998       );
 999       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
1000     }
1001
1002     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
1003       foreach ($matches as $key => $value) {
1004         $gid = substr($value, 21, -1);
1005         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
1006           "reset=1&gid={$gid}",
1007           TRUE, NULL, FALSE, TRUE
1008         );
1009         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
1010       }
1011     }
1012
1013     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
1014       foreach ($matches as $key => $value) {
1015         $gid = substr($value, 18, -1);
1016         $config = CRM_Core_Config::singleton();
1017         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
1018         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
1019         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
1020         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1021       }
1022     }
1023     return $str;
1024   }
1025
1026   /**
1027    * Replace welcome/confirmation tokens
1028    *
1029    * @param string $str
1030    *   The string with tokens to be replaced.
1031    * @param string $group
1032    *   The name of the group being subscribed.
1033    * @param bool $html
1034    *   Replace tokens with html or plain text.
1035    *
1036    * @return string
1037    *   The processed string
1038    */
1039   public static function &replaceWelcomeTokens($str, $group, $html) {
1040     if (self::token_match('welcome', 'group', $str)) {
1041       self::token_replace('welcome', 'group', $group, $str);
1042     }
1043     return $str;
1044   }
1045
1046   /**
1047    * Find unprocessed tokens (call this last)
1048    *
1049    * @param string $str
1050    *   The string to search.
1051    *
1052    * @return array
1053    *   Array of tokens that weren't replaced
1054    */
1055   public static function &unmatchedTokens(&$str) {
1056     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1057     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1058     return $match[1];
1059   }
1060
1061   /**
1062    * Find and replace tokens for each component.
1063    *
1064    * @param string $str
1065    *   The string to search.
1066    * @param array $contact
1067    *   Associative array of contact properties.
1068    * @param array $components
1069    *   A list of tokens that are known to exist in the email body.
1070    *
1071    * @param bool $escapeSmarty
1072    * @param bool $returnEmptyToken
1073    *
1074    * @return string
1075    *   The processed string
1076    */
1077   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1078     if (!is_array($components) || empty($contact)) {
1079       return $str;
1080     }
1081
1082     foreach ($components as $name => $tokens) {
1083       if (!is_array($tokens) || empty($tokens)) {
1084         continue;
1085       }
1086
1087       foreach ($tokens as $token) {
1088         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1089           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1090         }
1091         elseif (!$returnEmptyToken) {
1092           //replacing empty token
1093           self::token_replace($name, $token, "", $str, $escapeSmarty);
1094         }
1095       }
1096     }
1097     return $str;
1098   }
1099
1100   /**
1101    * Get array of string tokens.
1102    *
1103    * @param string $string
1104    *   The input string to parse for tokens.
1105    *
1106    * @return array
1107    *   array of tokens mentioned in field
1108    */
1109   public static function getTokens($string) {
1110     $matches = array();
1111     $tokens = array();
1112     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1113       $string,
1114       $matches,
1115       PREG_PATTERN_ORDER
1116     );
1117
1118     if ($matches[1]) {
1119       foreach ($matches[1] as $token) {
1120         list($type, $name) = preg_split('/\./', $token, 2);
1121         if ($name && $type) {
1122           if (!isset($tokens[$type])) {
1123             $tokens[$type] = array();
1124           }
1125           $tokens[$type][] = $name;
1126         }
1127       }
1128     }
1129     return $tokens;
1130   }
1131
1132   /**
1133    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1134    * and getTokens appears to be historical rather than intentional and should be reviewed
1135    * @param $string
1136    * @return array
1137    *   fields to pass in as return properties when populating token
1138    */
1139   public static function getReturnProperties(&$string) {
1140     $returnProperties = array();
1141     $matches = array();
1142     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1143       $string,
1144       $matches,
1145       PREG_PATTERN_ORDER
1146     );
1147     if ($matches[1]) {
1148       foreach ($matches[1] as $token) {
1149         list($type, $name) = preg_split('/\./', $token, 2);
1150         if ($name) {
1151           $returnProperties["{$name}"] = 1;
1152         }
1153       }
1154     }
1155
1156     return $returnProperties;
1157   }
1158
1159   /**
1160    * Gives required details of contacts in an indexed array format so we
1161    * can iterate in a nice loop and do token evaluation
1162    *
1163    * @param $contactIDs
1164    * @param array $returnProperties
1165    *   Of required properties.
1166    * @param bool $skipOnHold Don't return on_hold contact info also.
1167    *   Don't return on_hold contact info also.
1168    * @param bool $skipDeceased Don't return deceased contact info.
1169    *   Don't return deceased contact info.
1170    * @param array $extraParams
1171    *   Extra params.
1172    * @param array $tokens
1173    *   The list of tokens we've extracted from the content.
1174    * @param null $className
1175    * @param int $jobID
1176    *   The mailing list jobID - this is a legacy param.
1177    *
1178    * @return array
1179    */
1180   public static function getTokenDetails(
1181     $contactIDs,
1182     $returnProperties = NULL,
1183     $skipOnHold = TRUE,
1184     $skipDeceased = TRUE,
1185     $extraParams = NULL,
1186     $tokens = array(),
1187     $className = NULL,
1188     $jobID = NULL
1189   ) {
1190
1191     $params = array();
1192     foreach ($contactIDs as $key => $contactID) {
1193       $params[] = array(
1194         CRM_Core_Form::CB_PREFIX . $contactID,
1195         '=',
1196         1,
1197         0,
1198         0,
1199       );
1200     }
1201
1202     // fix for CRM-2613
1203     if ($skipDeceased) {
1204       $params[] = array('is_deceased', '=', 0, 0, 0);
1205     }
1206
1207     //fix for CRM-3798
1208     if ($skipOnHold) {
1209       $params[] = array('on_hold', '=', 0, 0, 0);
1210     }
1211
1212     if ($extraParams) {
1213       $params = array_merge($params, $extraParams);
1214     }
1215
1216     // if return properties are not passed then get all return properties
1217     if (empty($returnProperties)) {
1218       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1219         array('display_name', 'checksum', 'contact_id')
1220       );
1221       foreach ($fields as $key => $val) {
1222         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1223         // with contacts and are expensive to resolve.
1224         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1225         // and upstream calls to this should populate return properties.
1226         $unavailableFields = array('group', 'tag');
1227         if (!in_array($val, $unavailableFields)) {
1228           $returnProperties[$val] = 1;
1229         }
1230       }
1231     }
1232
1233     $custom = array();
1234     foreach ($returnProperties as $name => $dontCare) {
1235       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1236       if ($cfID) {
1237         $custom[] = $cfID;
1238       }
1239     }
1240
1241     $details = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1242
1243     $contactDetails = &$details[0];
1244
1245     foreach ($contactIDs as $key => $contactID) {
1246       if (array_key_exists($contactID, $contactDetails)) {
1247         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1248         ) {
1249           $communicationPreferences = array();
1250           foreach ($contactDetails[$contactID]['preferred_communication_method'] as $key => $val) {
1251             if ($val) {
1252               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1253             }
1254           }
1255           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1256         }
1257
1258         foreach ($custom as $cfID) {
1259           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1260             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1261           }
1262         }
1263
1264         // special case for greeting replacement
1265         foreach (array(
1266                    'email_greeting',
1267                    'postal_greeting',
1268                    'addressee',
1269                  ) as $val) {
1270           if (!empty($contactDetails[$contactID][$val])) {
1271             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1272           }
1273         }
1274       }
1275     }
1276
1277     // also call a hook and get token details
1278     CRM_Utils_Hook::tokenValues($details[0],
1279       $contactIDs,
1280       $jobID,
1281       $tokens,
1282       $className
1283     );
1284     return $details;
1285   }
1286
1287   /**
1288    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1289    * specific tokens to be rendered
1290    *
1291    * @param array $contactIDs
1292    *   This should always be array(0) or its not anonymous - left to keep signature same.
1293    *   as main fn
1294    * @param string $returnProperties
1295    * @param bool $skipOnHold
1296    * @param bool $skipDeceased
1297    * @param string $extraParams
1298    * @param array $tokens
1299    * @param string $className
1300    *   Sent as context to the hook.
1301    * @param string $jobID
1302    * @return array
1303    *   contactDetails with hooks swapped out
1304    */
1305   public static function getAnonymousTokenDetails($contactIDs = array(
1306       0,
1307     ),
1308                                            $returnProperties = NULL,
1309                                            $skipOnHold = TRUE,
1310                                            $skipDeceased = TRUE,
1311                                            $extraParams = NULL,
1312                                            $tokens = array(),
1313                                            $className = NULL,
1314                                            $jobID = NULL) {
1315     $details = array(0 => array());
1316     // also call a hook and get token details
1317     CRM_Utils_Hook::tokenValues($details[0],
1318       $contactIDs,
1319       $jobID,
1320       $tokens,
1321       $className
1322     );
1323     return $details;
1324   }
1325
1326   /**
1327    * Get Membership Token Details.
1328    * @param array $membershipIDs
1329    *   Array of membership IDS.
1330    */
1331   public static function getMembershipTokenDetails($membershipIDs) {
1332     $memberships = civicrm_api3('membership', 'get', array(
1333       'options' => array('limit' => 0),
1334       'membership_id' => array('IN' => (array) $membershipIDs),
1335     ));
1336     return $memberships['values'];
1337   }
1338
1339   /**
1340    * Replace existing greeting tokens in message/subject.
1341    *
1342    * This function operates by reference, modifying the first parameter. Other
1343    * methods for token replacement in this class return the modified string.
1344    * This leads to inconsistency in how these methods must be applied.
1345    *
1346    * @TODO Remove that inconsistency in usage.
1347    *
1348    * ::replaceContactTokens() may need to be called after this method, to
1349    * replace tokens supplied from this method.
1350    *
1351    * @param string $tokenString
1352    * @param array $contactDetails
1353    * @param int $contactId
1354    * @param string $className
1355    * @param bool $escapeSmarty
1356    */
1357   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1358
1359     if (!$contactDetails && !$contactId) {
1360       return;
1361     }
1362
1363     // check if there are any tokens
1364     $greetingTokens = self::getTokens($tokenString);
1365
1366     if (!empty($greetingTokens)) {
1367       // first use the existing contact object for token replacement
1368       if (!empty($contactDetails)) {
1369         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1370       }
1371
1372       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1373       // check if there are any unevaluated tokens
1374       $greetingTokens = self::getTokens($tokenString);
1375
1376       // $greetingTokens not empty, means there are few tokens which are not
1377       // evaluated, like custom data etc
1378       // so retrieve it from database
1379       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1380         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1381         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1382         $contactParams = array('contact_id' => $contactId);
1383
1384         $greetingDetails = self::getTokenDetails($contactParams,
1385           $greetingsReturnProperties,
1386           FALSE, FALSE, NULL,
1387           $greetingTokens,
1388           $className
1389         );
1390
1391         // again replace tokens
1392         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1393           $greetingDetails,
1394           TRUE,
1395           $greetingTokens,
1396           TRUE,
1397           $escapeSmarty
1398         );
1399       }
1400
1401       // check if there are still any unevaluated tokens
1402       $remainingTokens = self::getTokens($tokenString);
1403
1404       // $greetingTokens not empty, there are customized or hook tokens to replace
1405       if (!empty($remainingTokens)) {
1406         // Fill the return properties array
1407         $greetingTokens = $remainingTokens;
1408         reset($greetingTokens);
1409         $greetingsReturnProperties = array();
1410         while (list($key) = each($greetingTokens)) {
1411           $props = array_flip(CRM_Utils_Array::value($key, $greetingTokens));
1412           $props = array_fill_keys(array_keys($props), 1);
1413           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1414         }
1415         $contactParams = array('contact_id' => $contactId);
1416         $greetingDetails = self::getTokenDetails($contactParams,
1417           $greetingsReturnProperties,
1418           FALSE, FALSE, NULL,
1419           $greetingTokens,
1420           $className
1421         );
1422         // Prepare variables for calling replaceHookTokens
1423         $categories = array_keys($greetingTokens);
1424         list($contact) = $greetingDetails;
1425         // Replace tokens defined in Hooks.
1426         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1427       }
1428     }
1429   }
1430
1431   /**
1432    * At this point, $contactDetails has loaded the contact from the DAO. Any
1433    * (non-custom) missing fields are null.  By removing them, we can avoid
1434    * expensive calls to CRM_Contact_BAO_Query.
1435    *
1436    * @param string $tokenString
1437    * @param array $contactDetails
1438    */
1439   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1440     $greetingTokensOriginal = $greetingTokens;
1441     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1442     // Sometimes contactDetails are in a multidemensional array, sometimes a
1443     // single-dimension array.
1444     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1445       $contactDetails = current($contactDetails[0]);
1446     }
1447     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1448
1449     // Handle legacy tokens
1450     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1451       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1452         $nullFields[] = $oldToken;
1453       }
1454     }
1455
1456     // Remove null contact fields from $greetingTokens
1457     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1458
1459     // Also remove them from $tokenString
1460     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1461     // Handle legacy tokens again, sigh
1462     if (!empty($removedTokens)) {
1463       foreach ($removedTokens as $token) {
1464         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1465           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1466         }
1467       }
1468       foreach ($removedTokens as $token) {
1469         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1470       }
1471     }
1472   }
1473
1474   /**
1475    * @param $tokens
1476    *
1477    * @return array
1478    */
1479   public static function flattenTokens(&$tokens) {
1480     $flattenTokens = array();
1481
1482     foreach (array(
1483                'html',
1484                'text',
1485                'subject',
1486              ) as $prop) {
1487       if (!isset($tokens[$prop])) {
1488         continue;
1489       }
1490       foreach ($tokens[$prop] as $type => $names) {
1491         if (!isset($flattenTokens[$type])) {
1492           $flattenTokens[$type] = array();
1493         }
1494         foreach ($names as $name) {
1495           $flattenTokens[$type][$name] = 1;
1496         }
1497       }
1498     }
1499
1500     return $flattenTokens;
1501   }
1502
1503   /**
1504    * Replace all user tokens in $str
1505    *
1506    * @param string $str
1507    *   The string with tokens to be replaced.
1508    *
1509    * @param null $knownTokens
1510    * @param bool $escapeSmarty
1511    *
1512    * @return string
1513    *   The processed string
1514    */
1515   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1516     $key = 'user';
1517     if (!$knownTokens ||
1518       !isset($knownTokens[$key])
1519     ) {
1520       return $str;
1521     }
1522
1523     $str = preg_replace_callback(
1524       self::tokenRegex($key),
1525       function ($matches) use ($escapeSmarty) {
1526         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1527       },
1528       $str
1529     );
1530     return $str;
1531   }
1532
1533   /**
1534    * @param $token
1535    * @param bool $escapeSmarty
1536    *
1537    * @return string
1538    */
1539   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1540     $value = '';
1541
1542     list($objectName, $objectValue) = explode('-', $token, 2);
1543
1544     switch ($objectName) {
1545       case 'permission':
1546         $value = CRM_Core_Permission::permissionEmails($objectValue);
1547         break;
1548
1549       case 'role':
1550         $value = CRM_Core_Permission::roleEmails($objectValue);
1551         break;
1552     }
1553
1554     if ($escapeSmarty) {
1555       $value = self::tokenEscapeSmarty($value);
1556     }
1557
1558     return $value;
1559   }
1560
1561   protected static function _buildContributionTokens() {
1562     $key = 'contribution';
1563     if (self::$_tokens[$key] == NULL) {
1564       self::$_tokens[$key] = array_keys(array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1565         array('campaign', 'financial_type'),
1566         self::getCustomFieldTokens('Contribution')
1567       ));
1568     }
1569   }
1570
1571   /**
1572    * Store membership tokens on the static _tokens array.
1573    */
1574   protected static function _buildMembershipTokens() {
1575     $key = 'membership';
1576     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1577       $membershipTokens = array();
1578       $tokens = CRM_Core_SelectValues::membershipTokens();
1579       foreach ($tokens as $token => $dontCare) {
1580         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1581       }
1582       self::$_tokens[$key] = $membershipTokens;
1583     }
1584   }
1585
1586   /**
1587    * Replace tokens for an entity.
1588    * @param string $entity
1589    * @param array $entityArray
1590    *   (e.g. in format from api).
1591    * @param string $str
1592    *   String to replace in.
1593    * @param array $knownTokens
1594    *   Array of tokens present.
1595    * @param bool $escapeSmarty
1596    * @return string
1597    *   string with replacements made
1598    */
1599   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1600     if (!$knownTokens || empty($knownTokens[$entity])) {
1601       return $str;
1602     }
1603
1604     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1605     $fn = is_callable(array('CRM_Utils_Token', $fn)) ? $fn : 'getApiTokenReplacement';
1606     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1607     foreach ($knownTokens[$entity] as $token) {
1608       $replacement = self::$fn($entity, $token, $entityArray);
1609       if ($escapeSmarty) {
1610         $replacement = self::tokenEscapeSmarty($replacement);
1611       }
1612       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1613     }
1614     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1615   }
1616
1617   /**
1618    * @param int $caseId
1619    * @param int $str
1620    * @param array $knownTokens
1621    * @param bool $escapeSmarty
1622    * @return string
1623    * @throws \CiviCRM_API3_Exception
1624    */
1625   public static function replaceCaseTokens($caseId, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1626     if (!$knownTokens || empty($knownTokens['case'])) {
1627       return $str;
1628     }
1629     $case = civicrm_api3('case', 'getsingle', array('id' => $caseId));
1630     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1631   }
1632
1633   /**
1634    * Generic function for formatting token replacement for an api field
1635    *
1636    * @param string $entity
1637    * @param string $token
1638    * @param array $entityArray
1639    * @return string
1640    * @throws \CiviCRM_API3_Exception
1641    */
1642   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1643     if (!isset($entityArray[$token])) {
1644       return '';
1645     }
1646     $field = civicrm_api3($entity, 'getfield', array('action' => 'get', 'name' => $token, 'get_options' => 'get'));
1647     $field = $field['values'];
1648     $fieldType = CRM_Utils_Array::value('type', $field);
1649     // Boolean fields
1650     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1651       $field['options'] = array(ts('No'), ts('Yes'));
1652     }
1653     // Match pseudoconstants
1654     if (!empty($field['options'])) {
1655       $ret = array();
1656       foreach ((array) $entityArray[$token] as $val) {
1657         $ret[] = $field['options'][$val];
1658       }
1659       return implode(', ', $ret);
1660     }
1661     // Format date fields
1662     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1663       return CRM_Utils_Date::customFormat($entityArray[$token]);
1664     }
1665     return implode(', ', (array) $entityArray[$token]);
1666   }
1667
1668   /**
1669    * Replace Contribution tokens in html.
1670    *
1671    * @param string $str
1672    * @param array $contribution
1673    * @param bool|string $html
1674    * @param string $knownTokens
1675    * @param bool|string $escapeSmarty
1676    *
1677    * @return mixed
1678    */
1679   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1680     $key = 'contribution';
1681     if (!$knownTokens || !CRM_Utils_Array::value($key, $knownTokens)) {
1682       return $str; //early return
1683     }
1684     self::_buildContributionTokens();
1685
1686     // here we intersect with the list of pre-configured valid tokens
1687     // so that we remove anything we do not recognize
1688     // I hope to move this step out of here soon and
1689     // then we will just iterate on a list of tokens that are passed to us
1690
1691     $str = preg_replace_callback(
1692       self::tokenRegex($key),
1693       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1694         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1695       },
1696       $str
1697     );
1698
1699     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1700     return $str;
1701   }
1702
1703   /**
1704    * We have a situation where we are rendering more than one token in each field because we are combining
1705    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1706    *
1707    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1708    * it applies (other tokens are OK including date fields)
1709    *
1710    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1711    * does - but the fields is already being formatted through a date function
1712    *
1713    * @param string $separator
1714    * @param string $str
1715    * @param array $contribution
1716    * @param bool|string $html
1717    * @param string $knownTokens
1718    * @param bool|string $escapeSmarty
1719    *
1720    * @return string
1721    */
1722   public static function replaceMultipleContributionTokens($separator, $str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1723     if (empty($knownTokens['contribution'])) {
1724       return $str;
1725     }
1726
1727     if (in_array('receive_date', $knownTokens['contribution'])) {
1728       $formattedDates = array();
1729       $dates = explode($separator, $contribution['receive_date']);
1730       foreach ($dates as $date) {
1731         $formattedDates[] = CRM_Utils_Date::customFormat($date, NULL, array('j', 'm', 'Y'));
1732       }
1733       $str = str_replace("{contribution.receive_date}", implode($separator, $formattedDates), $str);
1734       unset($knownTokens['contribution']['receive_date']);
1735     }
1736     return self::replaceContributionTokens($str, $contribution, $html, $knownTokens, $escapeSmarty);
1737   }
1738
1739   /**
1740    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1741    * - this is used by the pdfLetter task from membership search
1742    * @param string $entity
1743    *   should always be "membership"
1744    * @param string $token
1745    *   field name
1746    * @param array $membership
1747    *   An api result array for a single membership.
1748    * @return string token replacement
1749    */
1750   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1751     self::_buildMembershipTokens();
1752     switch ($token) {
1753       case 'type':
1754         $value = $membership['membership_name'];
1755         break;
1756
1757       case 'status':
1758         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1759         $value = $statuses[$membership['status_id']];
1760         break;
1761
1762       case 'fee':
1763         try {
1764           $value = civicrm_api3('membership_type', 'getvalue', array(
1765               'id' => $membership['membership_type_id'],
1766               'return' => 'minimum_fee',
1767             ));
1768         }
1769         catch (CiviCRM_API3_Exception $e) {
1770           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1771           // api handles NULL (4.4)
1772           $value = 0;
1773         }
1774         break;
1775
1776       default:
1777         if (in_array($token, self::$_tokens[$entity])) {
1778           $value = $membership[$token];
1779           if (CRM_Utils_String::endsWith($token, '_date')) {
1780             $value = CRM_Utils_Date::customFormat($value);
1781           }
1782         }
1783         else {
1784           // ie unchanged
1785           $value = "{$entity}.{$token}";
1786         }
1787         break;
1788     }
1789
1790     return $value;
1791   }
1792
1793   /**
1794    * @param $token
1795    * @param $contribution
1796    * @param bool $html
1797    * @param bool $escapeSmarty
1798    *
1799    * @return mixed|string
1800    */
1801   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1802     self::_buildContributionTokens();
1803
1804     switch ($token) {
1805       case 'total_amount':
1806       case 'net_amount':
1807       case 'fee_amount':
1808       case 'non_deductible_amount':
1809         $value = CRM_Utils_Money::format(CRM_Utils_Array::retrieveValueRecursive($contribution, $token));
1810         break;
1811
1812       case 'receive_date':
1813         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1814         $value = CRM_Utils_Date::customFormat($value, NULL, array('j', 'm', 'Y'));
1815         break;
1816
1817       default:
1818         if (!in_array($token, self::$_tokens['contribution'])) {
1819           $value = "{contribution.$token}";
1820         }
1821         else {
1822           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1823         }
1824         break;
1825     }
1826
1827     if ($escapeSmarty) {
1828       $value = self::tokenEscapeSmarty($value);
1829     }
1830     return $value;
1831   }
1832
1833   /**
1834    * @return array
1835    *   [legacy_token => new_token]
1836    */
1837   public static function legacyContactTokens() {
1838     return array(
1839       'individual_prefix' => 'prefix_id',
1840       'individual_suffix' => 'suffix_id',
1841       'gender' => 'gender_id',
1842       'communication_style' => 'communication_style_id',
1843     );
1844   }
1845
1846   /**
1847    * Get all custom field tokens of $entity
1848    *
1849    * @param string $entity
1850    * @param bool $usedForTokenWidget
1851    *
1852    * @return array $customTokens
1853    *   return custom field tokens in array('custom_N' => 'label') format
1854    */
1855   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1856     $customTokens = array();
1857     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1858     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1859       $customTokens[sprintf($tokenName, $id)] = $info['label'];
1860     }
1861
1862     return $customTokens;
1863   }
1864
1865   /**
1866    * Formats a token list for the select2 widget
1867    *
1868    * @param $tokens
1869    * @return array
1870    */
1871   public static function formatTokensForDisplay($tokens) {
1872     $sorted = $output = array();
1873
1874     // Sort in ascending order by ignoring word case
1875     natcasesort($tokens);
1876
1877     // Attempt to place tokens into optgroups
1878     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1879     foreach ($tokens as $k => $v) {
1880       // Check to see if this token is already in a group e.g. for custom fields
1881       $split = explode(' :: ', $v);
1882       if (!empty($split[1])) {
1883         $sorted[$split[1]][] = array('id' => $k, 'text' => $split[0]);
1884       }
1885       // Group by entity
1886       else {
1887         $split = explode('.', trim($k, '{}'));
1888         if (isset($split[1])) {
1889           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucfirst($split[0]);
1890         }
1891         else {
1892           $entity = 'Contact';
1893         }
1894         $sorted[ts($entity)][] = array('id' => $k, 'text' => $v);
1895       }
1896     }
1897
1898     ksort($sorted);
1899     foreach ($sorted as $k => $v) {
1900       $output[] = array('text' => $k, 'children' => $v);
1901     }
1902
1903     return $output;
1904   }
1905
1906 }