CRM/Utils/Token.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33
  34 /**
  35  * Class to abstract token replacement.
  36  */
  37 class CRM_Utils_Token {
  38   static $_requiredTokens = NULL;
  39
  40   static $_tokens = array(
  41     'action' => array(
  42       'forward',
  43       'optOut',
  44       'optOutUrl',
  45       'reply',
  46       'unsubscribe',
  47       'unsubscribeUrl',
  48       'resubscribe',
  49       'resubscribeUrl',
  50       'subscribeUrl',
  51     ),
  52     'mailing' => array(
  53       'id',
  54       'name',
  55       'group',
  56       'subject',
  57       'viewUrl',
  58       'editUrl',
  59       'scheduleUrl',
  60       'approvalStatus',
  61       'approvalNote',
  62       'approveUrl',
  63       'creator',
  64       'creatorEmail',
  65     ),
  66     'user' => array(
  67       // we extract the stuff after the role / permission and return the
  68       // civicrm email addresses of all users with that role / permission
  69       // useful with rules integration
  70       'permission:',
  71       'role:',
  72     ),
  73     // populate this dynamically
  74     'contact' => NULL,
  75     // populate this dynamically
  76     'contribution' => NULL,
  77     'domain' => array(
  78       'name',
  79       'phone',
  80       'address',
  81       'email',
  82       'id',
  83       'description',
  84     ),
  85     'subscribe' => array('group'),
  86     'unsubscribe' => array('group'),
  87     'resubscribe' => array('group'),
  88     'welcome' => array('group'),
  89   );
  90
  91
  92   /**
  93    * @deprecated
  94    *   This is used by CiviMail but will be made redundant by FlexMailer.
  95    * @return array
  96    */
  97   public static function getRequiredTokens() {
  98     if (self::$_requiredTokens == NULL) {
  99       self::$_requiredTokens = array(
 100         'domain.address' => ts("Domain address - displays your organization's postal address."),
 101         'action.optOutUrl or action.unsubscribeUrl' => array(
 102           'action.optOut' => ts("'Opt out via email' - displays an email address for recipients to opt out of receiving emails from your organization."),
 103           'action.optOutUrl' => ts("'Opt out via web page' - creates a link for recipients to click if they want to opt out of receiving emails from your organization. Alternatively, you can include the 'Opt out via email' token."),
 104           'action.unsubscribe' => ts("'Unsubscribe via email' - displays an email address for recipients to unsubscribe from the specific mailing list used to send this message."),
 105           'action.unsubscribeUrl' => ts("'Unsubscribe via web page' - creates a link for recipients to unsubscribe from the specific mailing list used to send this message. Alternatively, you can include the 'Unsubscribe via email' token or one of the Opt-out tokens."),
 106         ),
 107       );
 108     }
 109     return self::$_requiredTokens;
 110   }
 111
 112   /**
 113    * Check a string (mailing body) for required tokens.
 114    *
 115    * @param string $str
 116    *   The message.
 117    *
 118    * @return bool|array
 119    *    true if all required tokens are found,
 120    *    else an array of the missing tokens
 121    */
 122   public static function requiredTokens(&$str) {
 123     // FlexMailer is a refactoring of CiviMail which provides new hooks/APIs/docs. If the sysadmin has opted to enable it, then use that instead of CiviMail.
 124     $requiredTokens = defined('CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS') ? Civi\Core\Resolver::singleton()->call(CIVICRM_FLEXMAILER_HACK_REQUIRED_TOKENS, array()) : CRM_Utils_Token::getRequiredTokens();
 125
 126     $missing = array();
 127     foreach ($requiredTokens as $token => $value) {
 128       if (!is_array($value)) {
 129         if (!preg_match('/(^|[^\{])' . preg_quote('{' . $token . '}') . '/', $str)) {
 130           $missing[$token] = $value;
 131         }
 132       }
 133       else {
 134         $present = FALSE;
 135         $desc = NULL;
 136         foreach ($value as $t => $d) {
 137           $desc = $d;
 138           if (preg_match('/(^|[^\{])' . preg_quote('{' . $t . '}') . '/', $str)) {
 139             $present = TRUE;
 140           }
 141         }
 142         if (!$present) {
 143           $missing[$token] = $desc;
 144         }
 145       }
 146     }
 147
 148     if (empty($missing)) {
 149       return TRUE;
 150     }
 151     return $missing;
 152   }
 153
 154   /**
 155    * Wrapper for token matching.
 156    *
 157    * @param string $type
 158    *   The token type (domain,mailing,contact,action).
 159    * @param string $var
 160    *   The token variable.
 161    * @param string $str
 162    *   The string to search.
 163    *
 164    * @return bool
 165    *   Was there a match
 166    */
 167   public static function token_match($type, $var, &$str) {
 168     $token = preg_quote('{' . "$type.$var") . '(\|.+?)?' . preg_quote('}');
 169     return preg_match("/(^|[^\{])$token/", $str);
 170   }
 171
 172   /**
 173    * Wrapper for token replacing.
 174    *
 175    * @param string $type
 176    *   The token type.
 177    * @param string $var
 178    *   The token variable.
 179    * @param string $value
 180    *   The value to substitute for the token.
 181    * @param string (reference) $str The string to replace in
 182    *
 183    * @param bool $escapeSmarty
 184    *
 185    * @return string
 186    *   The processed string
 187    */
 188   public static function &token_replace($type, $var, $value, &$str, $escapeSmarty = FALSE) {
 189     $token = preg_quote('{' . "$type.$var") . '(\|([^\}]+?))?' . preg_quote('}');
 190     if (!$value) {
 191       $value = '$3';
 192     }
 193     if ($escapeSmarty) {
 194       $value = self::tokenEscapeSmarty($value);
 195     }
 196     $str = preg_replace("/([^\{])?$token/", "\${1}$value", $str);
 197     return $str;
 198   }
 199
 200   /**
 201    * Get< the regex for token replacement
 202    *
 203    * @param string $token_type
 204    *   A string indicating the the type of token to be used in the expression.
 205    *
 206    * @return string
 207    *   regular expression sutiable for using in preg_replace
 208    */
 209   private static function tokenRegex($token_type) {
 210     return '/(?<!\{|\\\\)\{' . $token_type . '\.([\w]+(\-[\w\s]+)?)\}(?!\})/';
 211   }
 212
 213   /**
 214    * Escape the string so a malicious user cannot inject smarty code into the template.
 215    *
 216    * @param string $string
 217    *   A string that needs to be escaped from smarty parsing.
 218    *
 219    * @return string
 220    *   the escaped string
 221    */
 222   public static function tokenEscapeSmarty($string) {
 223     // need to use negative look-behind, as both str_replace() and preg_replace() are sequential
 224     return preg_replace(array('/{/', '/(?<!{ldelim)}/'), array('{ldelim}', '{rdelim}'), $string);
 225   }
 226
 227   /**
 228    * Replace all the domain-level tokens in $str
 229    *
 230    * @param string $str
 231    *   The string with tokens to be replaced.
 232    * @param object $domain
 233    *   The domain BAO.
 234    * @param bool $html
 235    *   Replace tokens with HTML or plain text.
 236    *
 237    * @param null $knownTokens
 238    * @param bool $escapeSmarty
 239    *
 240    * @return string
 241    *   The processed string
 242    */
 243   public static function &replaceDomainTokens(
 244     $str,
 245     &$domain,
 246     $html = FALSE,
 247     $knownTokens = NULL,
 248     $escapeSmarty = FALSE
 249   ) {
 250     $key = 'domain';
 251     if (
 252       !$knownTokens || empty($knownTokens[$key])
 253     ) {
 254       return $str;
 255     }
 256
 257     $str = preg_replace_callback(
 258       self::tokenRegex($key),
 259       function ($matches) use (&$domain, $html, $escapeSmarty) {
 260         return CRM_Utils_Token::getDomainTokenReplacement($matches[1], $domain, $html, $escapeSmarty);
 261       },
 262       $str
 263     );
 264     return $str;
 265   }
 266
 267   /**
 268    * @param $token
 269    * @param $domain
 270    * @param bool $html
 271    * @param bool $escapeSmarty
 272    *
 273    * @return mixed|null|string
 274    */
 275   public static function getDomainTokenReplacement($token, &$domain, $html = FALSE, $escapeSmarty = FALSE) {
 276     // check if the token we were passed is valid
 277     // we have to do this because this function is
 278     // called only when we find a token in the string
 279
 280     $loc = &$domain->getLocationValues();
 281
 282     if (!in_array($token, self::$_tokens['domain'])) {
 283       $value = "{domain.$token}";
 284     }
 285     elseif ($token == 'address') {
 286       static $addressCache = array();
 287
 288       $cache_key = $html ? 'address-html' : 'address-text';
 289       if (array_key_exists($cache_key, $addressCache)) {
 290         return $addressCache[$cache_key];
 291       }
 292
 293       $value = NULL;
 294       // Construct the address token
 295
 296       if (!empty($loc[$token])) {
 297         if ($html) {
 298           $value = $loc[$token][1]['display'];
 299           $value = str_replace("\n", '<br />', $value);
 300         }
 301         else {
 302           $value = $loc[$token][1]['display_text'];
 303         }
 304         $addressCache[$cache_key] = $value;
 305       }
 306     }
 307     elseif ($token == 'name' || $token == 'id' || $token == 'description') {
 308       $value = $domain->$token;
 309     }
 310     elseif ($token == 'phone' || $token == 'email') {
 311       // Construct the phone and email tokens
 312
 313       $value = NULL;
 314       if (!empty($loc[$token])) {
 315         foreach ($loc[$token] as $index => $entity) {
 316           $value = $entity[$token];
 317           break;
 318         }
 319       }
 320     }
 321
 322     if ($escapeSmarty) {
 323       $value = self::tokenEscapeSmarty($value);
 324     }
 325
 326     return $value;
 327   }
 328
 329   /**
 330    * Replace all the org-level tokens in $str
 331    *
 332    * @fixme: This function appears to be broken, as it depends on
 333    * nonexistant method: CRM_Core_BAO_CustomValue::getContactValues()
 334    * Marking as deprecated until this is fixed
 335    * @deprecated
 336    *
 337    * @param string $str
 338    *   The string with tokens to be replaced.
 339    * @param object $org
 340    *   Associative array of org properties.
 341    * @param bool $html
 342    *   Replace tokens with HTML or plain text.
 343    *
 344    * @param bool $escapeSmarty
 345    *
 346    * @return string
 347    *   The processed string
 348    */
 349   public static function &replaceOrgTokens($str, &$org, $html = FALSE, $escapeSmarty = FALSE) {
 350     self::$_tokens['org']
 351       = array_merge(
 352         array_keys(CRM_Contact_BAO_Contact::importableFields('Organization')),
 353         array('address', 'display_name', 'checksum', 'contact_id')
 354       );
 355
 356     $cv = NULL;
 357     foreach (self::$_tokens['org'] as $token) {
 358       // print "Getting token value for $token<br/><br/>";
 359       if ($token == '') {
 360         continue;
 361       }
 362
 363       // If the string doesn't contain this token, skip it.
 364
 365       if (!self::token_match('org', $token, $str)) {
 366         continue;
 367       }
 368
 369       // Construct value from $token and $contact
 370
 371       $value = NULL;
 372
 373       if ($cfID = CRM_Core_BAO_CustomField::getKeyID($token)) {
 374         // only generate cv if we need it
 375         if ($cv === NULL) {
 376           $cv = CRM_Core_BAO_CustomValue::getContactValues($org['contact_id']);
 377         }
 378         foreach ($cv as $cvFieldID => $value) {
 379           if ($cvFieldID == $cfID) {
 380             $value = CRM_Core_BAO_CustomField::displayValue($value, $cfID);
 381             break;
 382           }
 383         }
 384       }
 385       elseif ($token == 'checksum') {
 386         $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($org['contact_id']);
 387         $value = "cs={$cs}";
 388       }
 389       elseif ($token == 'address') {
 390         // Build the location values array
 391
 392         $loc = array();
 393         $loc['display_name'] = CRM_Utils_Array::retrieveValueRecursive($org, 'display_name');
 394         $loc['street_address'] = CRM_Utils_Array::retrieveValueRecursive($org, 'street_address');
 395         $loc['city'] = CRM_Utils_Array::retrieveValueRecursive($org, 'city');
 396         $loc['state_province'] = CRM_Utils_Array::retrieveValueRecursive($org, 'state_province');
 397         $loc['postal_code'] = CRM_Utils_Array::retrieveValueRecursive($org, 'postal_code');
 398
 399         // Construct the address token
 400
 401         $value = CRM_Utils_Address::format($loc);
 402         if ($html) {
 403           $value = str_replace("\n", '<br />', $value);
 404         }
 405       }
 406       else {
 407         $value = CRM_Utils_Array::retrieveValueRecursive($org, $token);
 408       }
 409
 410       self::token_replace('org', $token, $value, $str, $escapeSmarty);
 411     }
 412
 413     return $str;
 414   }
 415
 416   /**
 417    * Replace all mailing tokens in $str
 418    *
 419    * @param string $str
 420    *   The string with tokens to be replaced.
 421    * @param object $mailing
 422    *   The mailing BAO, or null for validation.
 423    * @param bool $html
 424    *   Replace tokens with HTML or plain text.
 425    *
 426    * @param null $knownTokens
 427    * @param bool $escapeSmarty
 428    *
 429    * @return string
 430    *   The processed string
 431    */
 432   public static function &replaceMailingTokens(
 433     $str,
 434     &$mailing,
 435     $html = FALSE,
 436     $knownTokens = NULL,
 437     $escapeSmarty = FALSE
 438   ) {
 439     $key = 'mailing';
 440     if (!$knownTokens || !isset($knownTokens[$key])) {
 441       return $str;
 442     }
 443
 444     $str = preg_replace_callback(
 445       self::tokenRegex($key),
 446       function ($matches) use (&$mailing, $escapeSmarty) {
 447         return CRM_Utils_Token::getMailingTokenReplacement($matches[1], $mailing, $escapeSmarty);
 448       },
 449       $str
 450     );
 451     return $str;
 452   }
 453
 454   /**
 455    * @param $token
 456    * @param $mailing
 457    * @param bool $escapeSmarty
 458    *
 459    * @return string
 460    */
 461   public static function getMailingTokenReplacement($token, &$mailing, $escapeSmarty = FALSE) {
 462     $value = '';
 463     switch ($token) {
 464       // CRM-7663
 465
 466       case 'id':
 467         $value = $mailing ? $mailing->id : 'undefined';
 468         break;
 469
 470       case 'name':
 471         $value = $mailing ? $mailing->name : 'Mailing Name';
 472         break;
 473
 474       case 'group':
 475         $groups = $mailing ? $mailing->getGroupNames() : array('Mailing Groups');
 476         $value = implode(', ', $groups);
 477         break;
 478
 479       case 'subject':
 480         $value = $mailing->subject;
 481         break;
 482
 483       case 'viewUrl':
 484         $mailingKey = $mailing->id;
 485         if ($hash = CRM_Mailing_BAO_Mailing::getMailingHash($mailingKey)) {
 486           $mailingKey = $hash;
 487         }
 488         $value = CRM_Utils_System::url('civicrm/mailing/view',
 489           "reset=1&id={$mailingKey}",
 490           TRUE, NULL, FALSE, TRUE
 491         );
 492         break;
 493
 494       case 'editUrl':
 495       case 'scheduleUrl':
 496         // Note: editUrl and scheduleUrl used to be different, but now there's
 497         // one screen which can adapt based on permissions (in workflow mode).
 498         $value = CRM_Utils_System::url('civicrm/mailing/send',
 499           "reset=1&mid={$mailing->id}&continue=true",
 500           TRUE, NULL, FALSE, TRUE
 501         );
 502         break;
 503
 504       case 'html':
 505         $page = new CRM_Mailing_Page_View();
 506         $value = $page->run($mailing->id, NULL, FALSE, TRUE);
 507         break;
 508
 509       case 'approvalStatus':
 510         $value = CRM_Core_PseudoConstant::getLabel('CRM_Mailing_DAO_Mailing', 'approval_status_id', $mailing->approval_status_id);
 511         break;
 512
 513       case 'approvalNote':
 514         $value = $mailing->approval_note;
 515         break;
 516
 517       case 'approveUrl':
 518         $value = CRM_Utils_System::url('civicrm/mailing/approve',
 519           "reset=1&mid={$mailing->id}",
 520           TRUE, NULL, FALSE, TRUE
 521         );
 522         break;
 523
 524       case 'creator':
 525         $value = CRM_Contact_BAO_Contact::displayName($mailing->created_id);
 526         break;
 527
 528       case 'creatorEmail':
 529         $value = CRM_Contact_BAO_Contact::getPrimaryEmail($mailing->created_id);
 530         break;
 531
 532       default:
 533         $value = "{mailing.$token}";
 534         break;
 535     }
 536
 537     if ($escapeSmarty) {
 538       $value = self::tokenEscapeSmarty($value);
 539     }
 540     return $value;
 541   }
 542
 543   /**
 544    * Replace all action tokens in $str
 545    *
 546    * @param string $str
 547    *   The string with tokens to be replaced.
 548    * @param array $addresses
 549    *   Assoc. array of VERP event addresses.
 550    * @param array $urls
 551    *   Assoc. array of action URLs.
 552    * @param bool $html
 553    *   Replace tokens with HTML or plain text.
 554    * @param array $knownTokens
 555    *   A list of tokens that are known to exist in the email body.
 556    *
 557    * @param bool $escapeSmarty
 558    *
 559    * @return string
 560    *   The processed string
 561    */
 562   public static function &replaceActionTokens(
 563     $str,
 564     &$addresses,
 565     &$urls,
 566     $html = FALSE,
 567     $knownTokens = NULL,
 568     $escapeSmarty = FALSE
 569   ) {
 570     $key = 'action';
 571     // here we intersect with the list of pre-configured valid tokens
 572     // so that we remove anything we do not recognize
 573     // I hope to move this step out of here soon and
 574     // then we will just iterate on a list of tokens that are passed to us
 575     if (!$knownTokens || empty($knownTokens[$key])) {
 576       return $str;
 577     }
 578
 579     $str = preg_replace_callback(
 580       self::tokenRegex($key),
 581       function ($matches) use (&$addresses, &$urls, $html, $escapeSmarty) {
 582         return CRM_Utils_Token::getActionTokenReplacement($matches[1], $addresses, $urls, $html, $escapeSmarty);
 583       },
 584       $str
 585     );
 586     return $str;
 587   }
 588
 589   /**
 590    * @param $token
 591    * @param $addresses
 592    * @param $urls
 593    * @param bool $html
 594    * @param bool $escapeSmarty
 595    *
 596    * @return mixed|string
 597    */
 598   public static function getActionTokenReplacement(
 599     $token,
 600     &$addresses,
 601     &$urls,
 602     $html = FALSE,
 603     $escapeSmarty = FALSE
 604   ) {
 605     // If the token is an email action, use it.  Otherwise, find the
 606     // appropriate URL.
 607
 608     if (!in_array($token, self::$_tokens['action'])) {
 609       $value = "{action.$token}";
 610     }
 611     else {
 612       $value = CRM_Utils_Array::value($token, $addresses);
 613
 614       if ($value == NULL) {
 615         $value = CRM_Utils_Array::value($token, $urls);
 616       }
 617
 618       if ($value && $html) {
 619         // fix for CRM-2318
 620         if ((substr($token, -3) != 'Url') && ($token != 'forward')) {
 621           $value = "mailto:$value";
 622         }
 623       }
 624       elseif ($value && !$html) {
 625         $value = str_replace('&amp;', '&', $value);
 626       }
 627     }
 628
 629     if ($escapeSmarty) {
 630       $value = self::tokenEscapeSmarty($value);
 631     }
 632     return $value;
 633   }
 634
 635   /**
 636    * Replace all the contact-level tokens in $str with information from
 637    * $contact.
 638    *
 639    * @param string $str
 640    *   The string with tokens to be replaced.
 641    * @param array $contact
 642    *   Associative array of contact properties.
 643    * @param bool $html
 644    *   Replace tokens with HTML or plain text.
 645    * @param array $knownTokens
 646    *   A list of tokens that are known to exist in the email body.
 647    * @param bool $returnBlankToken
 648    *   Return unevaluated token if value is null.
 649    *
 650    * @param bool $escapeSmarty
 651    *
 652    * @return string
 653    *   The processed string
 654    */
 655   public static function &replaceContactTokens(
 656     $str,
 657     &$contact,
 658     $html = FALSE,
 659     $knownTokens = NULL,
 660     $returnBlankToken = FALSE,
 661     $escapeSmarty = FALSE
 662   ) {
 663     // Refresh contact tokens in case they have changed. There is heavy caching
 664     // in exportable fields so there is no benefit in doing this conditionally.
 665     self::$_tokens['contact'] = array_merge(
 666       array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 667       array('checksum', 'contact_id')
 668     );
 669
 670     $key = 'contact';
 671     // here we intersect with the list of pre-configured valid tokens
 672     // so that we remove anything we do not recognize
 673     // I hope to move this step out of here soon and
 674     // then we will just iterate on a list of tokens that are passed to us
 675     if (!$knownTokens || empty($knownTokens[$key])) {
 676       return $str;
 677     }
 678
 679     $str = preg_replace_callback(
 680       self::tokenRegex($key),
 681       function ($matches) use (&$contact, $html, $returnBlankToken, $escapeSmarty) {
 682         return CRM_Utils_Token::getContactTokenReplacement($matches[1], $contact, $html, $returnBlankToken, $escapeSmarty);
 683       },
 684       $str
 685     );
 686
 687     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
 688     return $str;
 689   }
 690
 691   /**
 692    * @param $token
 693    * @param $contact
 694    * @param bool $html
 695    * @param bool $returnBlankToken
 696    * @param bool $escapeSmarty
 697    *
 698    * @return bool|mixed|null|string
 699    */
 700   public static function getContactTokenReplacement(
 701     $token,
 702     &$contact,
 703     $html = FALSE,
 704     $returnBlankToken = FALSE,
 705     $escapeSmarty = FALSE
 706   ) {
 707     if (self::$_tokens['contact'] == NULL) {
 708       /* This should come from UF */
 709
 710       self::$_tokens['contact']
 711         = array_merge(
 712           array_keys(CRM_Contact_BAO_Contact::exportableFields('All')),
 713           array('checksum', 'contact_id')
 714         );
 715     }
 716
 717     // Construct value from $token and $contact
 718
 719     $value = NULL;
 720     $noReplace = FALSE;
 721
 722     // Support legacy tokens
 723     $token = CRM_Utils_Array::value($token, self::legacyContactTokens(), $token);
 724
 725     // check if the token we were passed is valid
 726     // we have to do this because this function is
 727     // called only when we find a token in the string
 728
 729     if (!in_array($token, self::$_tokens['contact'])) {
 730       $noReplace = TRUE;
 731     }
 732     elseif ($token == 'checksum') {
 733       $hash = CRM_Utils_Array::value('hash', $contact);
 734       $contactID = CRM_Utils_Array::retrieveValueRecursive($contact, 'contact_id');
 735       $cs = CRM_Contact_BAO_Contact_Utils::generateChecksum($contactID,
 736         NULL,
 737         NULL,
 738         $hash
 739       );
 740       $value = "cs={$cs}";
 741     }
 742     else {
 743       $value = CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 744
 745       // FIXME: for some pseudoconstants we get array ( 0 => id, 1 => label )
 746       if (is_array($value)) {
 747         $value = $value[1];
 748       }
 749       // Convert pseudoconstants using metadata
 750       elseif ($value && is_numeric($value)) {
 751         $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
 752         if (!empty($allFields[$token]['pseudoconstant'])) {
 753           $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
 754         }
 755       }
 756       elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
 757         $value = CRM_Utils_Date::customFormat($value);
 758       }
 759     }
 760
 761     if (!$html) {
 762       $value = str_replace('&amp;', '&', $value);
 763     }
 764
 765     // if null then return actual token
 766     if ($returnBlankToken && !$value) {
 767       $noReplace = TRUE;
 768     }
 769
 770     if ($noReplace) {
 771       $value = "{contact.$token}";
 772     }
 773
 774     if ($escapeSmarty
 775       && !($returnBlankToken && $noReplace)
 776     ) {
 777       // $returnBlankToken means the caller wants to do further attempts at
 778       // processing unreplaced tokens -- so don't escape them yet in this case.
 779       $value = self::tokenEscapeSmarty($value);
 780     }
 781
 782     return $value;
 783   }
 784
 785   /**
 786    * Replace all the hook tokens in $str with information from
 787    * $contact.
 788    *
 789    * @param string $str
 790    *   The string with tokens to be replaced.
 791    * @param array $contact
 792    *   Associative array of contact properties (including hook token values).
 793    * @param $categories
 794    * @param bool $html
 795    *   Replace tokens with HTML or plain text.
 796    *
 797    * @param bool $escapeSmarty
 798    *
 799    * @return string
 800    *   The processed string
 801    */
 802   public static function &replaceHookTokens(
 803     $str,
 804     &$contact,
 805     &$categories,
 806     $html = FALSE,
 807     $escapeSmarty = FALSE
 808   ) {
 809     foreach ($categories as $key) {
 810       $str = preg_replace_callback(
 811         self::tokenRegex($key),
 812         function ($matches) use (&$contact, $key, $html, $escapeSmarty) {
 813           return CRM_Utils_Token::getHookTokenReplacement($matches[1], $contact, $key, $html, $escapeSmarty);
 814         },
 815         $str
 816       );
 817     }
 818     return $str;
 819   }
 820
 821   /**
 822    * Parse html through Smarty resolving any smarty functions.
 823    * @param string $tokenHtml
 824    * @param array $entity
 825    * @param string $entityType
 826    * @return string
 827    *   html parsed through smarty
 828    */
 829   public static function parseThroughSmarty($tokenHtml, $entity, $entityType = 'contact') {
 830     if (defined('CIVICRM_MAIL_SMARTY') && CIVICRM_MAIL_SMARTY) {
 831       $smarty = CRM_Core_Smarty::singleton();
 832       // also add the tokens to the template
 833       $smarty->assign_by_ref($entityType, $entity);
 834       $tokenHtml = $smarty->fetch("string:$tokenHtml");
 835     }
 836     return $tokenHtml;
 837   }
 838
 839   /**
 840    * @param $token
 841    * @param $contact
 842    * @param $category
 843    * @param bool $html
 844    * @param bool $escapeSmarty
 845    *
 846    * @return mixed|string
 847    */
 848   public static function getHookTokenReplacement(
 849     $token,
 850     &$contact,
 851     $category,
 852     $html = FALSE,
 853     $escapeSmarty = FALSE
 854   ) {
 855     $value = CRM_Utils_Array::value("{$category}.{$token}", $contact);
 856
 857     if ($value && !$html) {
 858       $value = str_replace('&amp;', '&', $value);
 859     }
 860
 861     if ($escapeSmarty) {
 862       $value = self::tokenEscapeSmarty($value);
 863     }
 864
 865     return $value;
 866   }
 867
 868   /**
 869    * unescapeTokens removes any characters that caused the replacement routines to skip token replacement
 870    * for example {{token}}  or \{token}  will result in {token} in the final email
 871    *
 872    * this routine will remove the extra backslashes and braces
 873    *
 874    * @param $str ref to the string that will be scanned and modified
 875    */
 876   public static function unescapeTokens(&$str) {
 877     $str = preg_replace('/\\\\|\{(\{\w+\.\w+\})\}/', '\\1', $str);
 878   }
 879
 880   /**
 881    * Replace unsubscribe tokens.
 882    *
 883    * @param string $str
 884    *   The string with tokens to be replaced.
 885    * @param object $domain
 886    *   The domain BAO.
 887    * @param array $groups
 888    *   The groups (if any) being unsubscribed.
 889    * @param bool $html
 890    *   Replace tokens with html or plain text.
 891    * @param int $contact_id
 892    *   The contact ID.
 893    * @param string $hash The security hash of the unsub event
 894    *
 895    * @return string
 896    *   The processed string
 897    */
 898   public static function &replaceUnsubscribeTokens(
 899     $str,
 900     &$domain,
 901     &$groups,
 902     $html,
 903     $contact_id,
 904     $hash
 905   ) {
 906     if (self::token_match('unsubscribe', 'group', $str)) {
 907       if (!empty($groups)) {
 908         $config = CRM_Core_Config::singleton();
 909         $base = CRM_Utils_System::baseURL();
 910
 911         // FIXME: an ugly hack for CRM-2035, to be dropped once CRM-1799 is implemented
 912         $dao = new CRM_Contact_DAO_Group();
 913         $dao->find();
 914         while ($dao->fetch()) {
 915           if (substr($dao->visibility, 0, 6) == 'Public') {
 916             $visibleGroups[] = $dao->id;
 917           }
 918         }
 919         $value = implode(', ', $groups);
 920         self::token_replace('unsubscribe', 'group', $value, $str);
 921       }
 922     }
 923     return $str;
 924   }
 925
 926   /**
 927    * Replace resubscribe tokens.
 928    *
 929    * @param string $str
 930    *   The string with tokens to be replaced.
 931    * @param object $domain
 932    *   The domain BAO.
 933    * @param array $groups
 934    *   The groups (if any) being resubscribed.
 935    * @param bool $html
 936    *   Replace tokens with html or plain text.
 937    * @param int $contact_id
 938    *   The contact ID.
 939    * @param string $hash The security hash of the resub event
 940    *
 941    * @return string
 942    *   The processed string
 943    */
 944   public static function &replaceResubscribeTokens(
 945     $str, &$domain, &$groups, $html,
 946     $contact_id, $hash
 947   ) {
 948     if (self::token_match('resubscribe', 'group', $str)) {
 949       if (!empty($groups)) {
 950         $value = implode(', ', $groups);
 951         self::token_replace('resubscribe', 'group', $value, $str);
 952       }
 953     }
 954     return $str;
 955   }
 956
 957   /**
 958    * Replace subscription-confirmation-request tokens
 959    *
 960    * @param string $str
 961    *   The string with tokens to be replaced.
 962    * @param string $group
 963    *   The name of the group being subscribed.
 964    * @param $url
 965    * @param bool $html
 966    *   Replace tokens with html or plain text.
 967    *
 968    * @return string
 969    *   The processed string
 970    */
 971   public static function &replaceSubscribeTokens($str, $group, $url, $html) {
 972     if (self::token_match('subscribe', 'group', $str)) {
 973       self::token_replace('subscribe', 'group', $group, $str);
 974     }
 975     if (self::token_match('subscribe', 'url', $str)) {
 976       self::token_replace('subscribe', 'url', $url, $str);
 977     }
 978     return $str;
 979   }
 980
 981   /**
 982    * Replace subscription-invitation tokens
 983    *
 984    * @param string $str
 985    *   The string with tokens to be replaced.
 986    *
 987    * @return string
 988    *   The processed string
 989    */
 990   public static function &replaceSubscribeInviteTokens($str) {
 991     if (preg_match('/\{action\.subscribeUrl\}/', $str)) {
 992       $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
 993         'reset=1',
 994         TRUE, NULL, FALSE, TRUE
 995       );
 996       $str = preg_replace('/\{action\.subscribeUrl\}/', $url, $str);
 997     }
 998
 999     if (preg_match('/\{action\.subscribeUrl.\d+\}/', $str, $matches)) {
1000       foreach ($matches as $key => $value) {
1001         $gid = substr($value, 21, -1);
1002         $url = CRM_Utils_System::url('civicrm/mailing/subscribe',
1003           "reset=1&gid={$gid}",
1004           TRUE, NULL, FALSE, TRUE
1005         );
1006         $str = preg_replace('/' . preg_quote($value) . '/', $url, $str);
1007       }
1008     }
1009
1010     if (preg_match('/\{action\.subscribe.\d+\}/', $str, $matches)) {
1011       foreach ($matches as $key => $value) {
1012         $gid = substr($value, 18, -1);
1013         $config = CRM_Core_Config::singleton();
1014         $domain = CRM_Core_BAO_MailSettings::defaultDomain();
1015         $localpart = CRM_Core_BAO_MailSettings::defaultLocalpart();
1016         // we add the 0.0000000000000000 part to make this match the other email patterns (with action, two ids and a hash)
1017         $str = preg_replace('/' . preg_quote($value) . '/', "mailto:{$localpart}s.{$gid}.0.0000000000000000@$domain", $str);
1018       }
1019     }
1020     return $str;
1021   }
1022
1023   /**
1024    * Replace welcome/confirmation tokens
1025    *
1026    * @param string $str
1027    *   The string with tokens to be replaced.
1028    * @param string $group
1029    *   The name of the group being subscribed.
1030    * @param bool $html
1031    *   Replace tokens with html or plain text.
1032    *
1033    * @return string
1034    *   The processed string
1035    */
1036   public static function &replaceWelcomeTokens($str, $group, $html) {
1037     if (self::token_match('welcome', 'group', $str)) {
1038       self::token_replace('welcome', 'group', $group, $str);
1039     }
1040     return $str;
1041   }
1042
1043   /**
1044    * Find unprocessed tokens (call this last)
1045    *
1046    * @param string $str
1047    *   The string to search.
1048    *
1049    * @return array
1050    *   Array of tokens that weren't replaced
1051    */
1052   public static function &unmatchedTokens(&$str) {
1053     //preg_match_all('/[^\{\\\\]\{(\w+\.\w+)\}[^\}]/', $str, $match);
1054     preg_match_all('/\{(\w+\.\w+)\}/', $str, $match);
1055     return $match[1];
1056   }
1057
1058   /**
1059    * Find and replace tokens for each component.
1060    *
1061    * @param string $str
1062    *   The string to search.
1063    * @param array $contact
1064    *   Associative array of contact properties.
1065    * @param array $components
1066    *   A list of tokens that are known to exist in the email body.
1067    *
1068    * @param bool $escapeSmarty
1069    * @param bool $returnEmptyToken
1070    *
1071    * @return string
1072    *   The processed string
1073    */
1074   public static function &replaceComponentTokens(&$str, $contact, $components, $escapeSmarty = FALSE, $returnEmptyToken = TRUE) {
1075     if (!is_array($components) || empty($contact)) {
1076       return $str;
1077     }
1078
1079     foreach ($components as $name => $tokens) {
1080       if (!is_array($tokens) || empty($tokens)) {
1081         continue;
1082       }
1083
1084       foreach ($tokens as $token) {
1085         if (self::token_match($name, $token, $str) && isset($contact[$name . '.' . $token])) {
1086           self::token_replace($name, $token, $contact[$name . '.' . $token], $str, $escapeSmarty);
1087         }
1088         elseif (!$returnEmptyToken) {
1089           //replacing empty token
1090           self::token_replace($name, $token, "", $str, $escapeSmarty);
1091         }
1092       }
1093     }
1094     return $str;
1095   }
1096
1097   /**
1098    * Get array of string tokens.
1099    *
1100    * @param string $string
1101    *   The input string to parse for tokens.
1102    *
1103    * @return array
1104    *   array of tokens mentioned in field
1105    */
1106   public static function getTokens($string) {
1107     $matches = array();
1108     $tokens = array();
1109     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1110       $string,
1111       $matches,
1112       PREG_PATTERN_ORDER
1113     );
1114
1115     if ($matches[1]) {
1116       foreach ($matches[1] as $token) {
1117         list($type, $name) = preg_split('/\./', $token, 2);
1118         if ($name && $type) {
1119           if (!isset($tokens[$type])) {
1120             $tokens[$type] = array();
1121           }
1122           $tokens[$type][] = $name;
1123         }
1124       }
1125     }
1126     return $tokens;
1127   }
1128
1129   /**
1130    * Function to determine which values to retrieve to insert into tokens. The heavy resemblance between this function
1131    * and getTokens appears to be historical rather than intentional and should be reviewed
1132    * @param $string
1133    * @return array
1134    *   fields to pass in as return properties when populating token
1135    */
1136   public static function getReturnProperties(&$string) {
1137     $returnProperties = array();
1138     $matches = array();
1139     preg_match_all('/(?<!\{|\\\\)\{(\w+\.\w+)\}(?!\})/',
1140       $string,
1141       $matches,
1142       PREG_PATTERN_ORDER
1143     );
1144     if ($matches[1]) {
1145       foreach ($matches[1] as $token) {
1146         list($type, $name) = preg_split('/\./', $token, 2);
1147         if ($name) {
1148           $returnProperties["{$name}"] = 1;
1149         }
1150       }
1151     }
1152
1153     return $returnProperties;
1154   }
1155
1156   /**
1157    * Gives required details of contacts in an indexed array format so we
1158    * can iterate in a nice loop and do token evaluation
1159    *
1160    * @param array $contactIDs
1161    * @param array $returnProperties
1162    *   Of required properties.
1163    * @param bool $skipOnHold Don't return on_hold contact info also.
1164    *   Don't return on_hold contact info also.
1165    * @param bool $skipDeceased Don't return deceased contact info.
1166    *   Don't return deceased contact info.
1167    * @param array $extraParams
1168    *   Extra params.
1169    * @param array $tokens
1170    *   The list of tokens we've extracted from the content.
1171    * @param null $className
1172    * @param int $jobID
1173    *   The mailing list jobID - this is a legacy param.
1174    *
1175    * @return array
1176    */
1177   public static function getTokenDetails(
1178     $contactIDs,
1179     $returnProperties = NULL,
1180     $skipOnHold = TRUE,
1181     $skipDeceased = TRUE,
1182     $extraParams = NULL,
1183     $tokens = array(),
1184     $className = NULL,
1185     $jobID = NULL
1186   ) {
1187
1188     $params = array();
1189     foreach ($contactIDs as $contactID) {
1190       $params[] = array(
1191         CRM_Core_Form::CB_PREFIX . $contactID,
1192         '=',
1193         1,
1194         0,
1195         0,
1196       );
1197     }
1198
1199     // fix for CRM-2613
1200     if ($skipDeceased) {
1201       $params[] = array('is_deceased', '=', 0, 0, 0);
1202     }
1203
1204     //fix for CRM-3798
1205     if ($skipOnHold) {
1206       $params[] = array('on_hold', '=', 0, 0, 0);
1207     }
1208
1209     if ($extraParams) {
1210       $params = array_merge($params, $extraParams);
1211     }
1212
1213     // if return properties are not passed then get all return properties
1214     if (empty($returnProperties)) {
1215       $fields = array_merge(array_keys(CRM_Contact_BAO_Contact::exportableFields()),
1216         array('display_name', 'checksum', 'contact_id')
1217       );
1218       foreach ($fields as $val) {
1219         // The unavailable fields are not available as tokens, do not have a one-2-one relationship
1220         // with contacts and are expensive to resolve.
1221         // @todo see CRM-17253 - there are some other fields (e.g note) that should be excluded
1222         // and upstream calls to this should populate return properties.
1223         $unavailableFields = array('group', 'tag');
1224         if (!in_array($val, $unavailableFields)) {
1225           $returnProperties[$val] = 1;
1226         }
1227       }
1228     }
1229
1230     $custom = array();
1231     foreach ($returnProperties as $name => $dontCare) {
1232       $cfID = CRM_Core_BAO_CustomField::getKeyID($name);
1233       if ($cfID) {
1234         $custom[] = $cfID;
1235       }
1236     }
1237
1238     $details = CRM_Contact_BAO_Query::apiQuery($params, $returnProperties, NULL, NULL, 0, count($contactIDs), TRUE, FALSE, TRUE, CRM_Contact_BAO_Query::MODE_CONTACTS, NULL, TRUE);
1239
1240     $contactDetails = &$details[0];
1241
1242     foreach ($contactIDs as $contactID) {
1243       if (array_key_exists($contactID, $contactDetails)) {
1244         if (!empty($contactDetails[$contactID]['preferred_communication_method'])
1245         ) {
1246           $communicationPreferences = array();
1247           foreach ($contactDetails[$contactID]['preferred_communication_method'] as $val) {
1248             if ($val) {
1249               $communicationPreferences[$val] = CRM_Core_PseudoConstant::getLabel('CRM_Contact_DAO_Contact', 'preferred_communication_method', $val);
1250             }
1251           }
1252           $contactDetails[$contactID]['preferred_communication_method'] = implode(', ', $communicationPreferences);
1253         }
1254
1255         foreach ($custom as $cfID) {
1256           if (isset($contactDetails[$contactID]["custom_{$cfID}"])) {
1257             $contactDetails[$contactID]["custom_{$cfID}"] = CRM_Core_BAO_CustomField::displayValue($contactDetails[$contactID]["custom_{$cfID}"], $cfID);
1258           }
1259         }
1260
1261         // special case for greeting replacement
1262         foreach (array(
1263                    'email_greeting',
1264                    'postal_greeting',
1265                    'addressee',
1266                  ) as $val) {
1267           if (!empty($contactDetails[$contactID][$val])) {
1268             $contactDetails[$contactID][$val] = $contactDetails[$contactID]["{$val}_display"];
1269           }
1270         }
1271       }
1272     }
1273
1274     // also call a hook and get token details
1275     CRM_Utils_Hook::tokenValues($details[0],
1276       $contactIDs,
1277       $jobID,
1278       $tokens,
1279       $className
1280     );
1281     return $details;
1282   }
1283
1284   /**
1285    * Call hooks on tokens for anonymous users - contact id is set to 0 - this allows non-contact
1286    * specific tokens to be rendered
1287    *
1288    * @param array $contactIDs
1289    *   This should always be array(0) or its not anonymous - left to keep signature same.
1290    *   as main fn
1291    * @param string $returnProperties
1292    * @param bool $skipOnHold
1293    * @param bool $skipDeceased
1294    * @param string $extraParams
1295    * @param array $tokens
1296    * @param string $className
1297    *   Sent as context to the hook.
1298    * @param string $jobID
1299    * @return array
1300    *   contactDetails with hooks swapped out
1301    */
1302   public static function getAnonymousTokenDetails($contactIDs = array(
1303       0,
1304     ),
1305                                            $returnProperties = NULL,
1306                                            $skipOnHold = TRUE,
1307                                            $skipDeceased = TRUE,
1308                                            $extraParams = NULL,
1309                                            $tokens = array(),
1310                                            $className = NULL,
1311                                            $jobID = NULL) {
1312     $details = array(0 => array());
1313     // also call a hook and get token details
1314     CRM_Utils_Hook::tokenValues($details[0],
1315       $contactIDs,
1316       $jobID,
1317       $tokens,
1318       $className
1319     );
1320     return $details;
1321   }
1322
1323   /**
1324    * Get Membership Token Details.
1325    * @param array $membershipIDs
1326    *   Array of membership IDS.
1327    */
1328   public static function getMembershipTokenDetails($membershipIDs) {
1329     $memberships = civicrm_api3('membership', 'get', array(
1330       'options' => array('limit' => 0),
1331       'membership_id' => array('IN' => (array) $membershipIDs),
1332     ));
1333     return $memberships['values'];
1334   }
1335
1336   /**
1337    * Replace existing greeting tokens in message/subject.
1338    *
1339    * This function operates by reference, modifying the first parameter. Other
1340    * methods for token replacement in this class return the modified string.
1341    * This leads to inconsistency in how these methods must be applied.
1342    *
1343    * @TODO Remove that inconsistency in usage.
1344    *
1345    * ::replaceContactTokens() may need to be called after this method, to
1346    * replace tokens supplied from this method.
1347    *
1348    * @param string $tokenString
1349    * @param array $contactDetails
1350    * @param int $contactId
1351    * @param string $className
1352    * @param bool $escapeSmarty
1353    */
1354   public static function replaceGreetingTokens(&$tokenString, $contactDetails = NULL, $contactId = NULL, $className = NULL, $escapeSmarty = FALSE) {
1355
1356     if (!$contactDetails && !$contactId) {
1357       return;
1358     }
1359
1360     // check if there are any tokens
1361     $greetingTokens = self::getTokens($tokenString);
1362
1363     if (!empty($greetingTokens)) {
1364       // first use the existing contact object for token replacement
1365       if (!empty($contactDetails)) {
1366         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString, $contactDetails, TRUE, $greetingTokens, TRUE, $escapeSmarty);
1367       }
1368
1369       self::removeNullContactTokens($tokenString, $contactDetails, $greetingTokens);
1370       // check if there are any unevaluated tokens
1371       $greetingTokens = self::getTokens($tokenString);
1372
1373       // $greetingTokens not empty, means there are few tokens which are not
1374       // evaluated, like custom data etc
1375       // so retrieve it from database
1376       if (!empty($greetingTokens) && array_key_exists('contact', $greetingTokens)) {
1377         $greetingsReturnProperties = array_flip(CRM_Utils_Array::value('contact', $greetingTokens));
1378         $greetingsReturnProperties = array_fill_keys(array_keys($greetingsReturnProperties), 1);
1379         $contactParams = array('contact_id' => $contactId);
1380
1381         $greetingDetails = self::getTokenDetails($contactParams,
1382           $greetingsReturnProperties,
1383           FALSE, FALSE, NULL,
1384           $greetingTokens,
1385           $className
1386         );
1387
1388         // again replace tokens
1389         $tokenString = CRM_Utils_Token::replaceContactTokens($tokenString,
1390           $greetingDetails,
1391           TRUE,
1392           $greetingTokens,
1393           TRUE,
1394           $escapeSmarty
1395         );
1396       }
1397
1398       // check if there are still any unevaluated tokens
1399       $remainingTokens = self::getTokens($tokenString);
1400
1401       // $greetingTokens not empty, there are customized or hook tokens to replace
1402       if (!empty($remainingTokens)) {
1403         // Fill the return properties array
1404         $greetingTokens = $remainingTokens;
1405         reset($greetingTokens);
1406         $greetingsReturnProperties = array();
1407         foreach ($greetingTokens as $value) {
1408           $props = array_flip($value);
1409           $props = array_fill_keys(array_keys($props), 1);
1410           $greetingsReturnProperties = $greetingsReturnProperties + $props;
1411         }
1412         $contactParams = array('contact_id' => $contactId);
1413         $greetingDetails = self::getTokenDetails($contactParams,
1414           $greetingsReturnProperties,
1415           FALSE, FALSE, NULL,
1416           $greetingTokens,
1417           $className
1418         );
1419         // Prepare variables for calling replaceHookTokens
1420         $categories = array_keys($greetingTokens);
1421         list($contact) = $greetingDetails;
1422         // Replace tokens defined in Hooks.
1423         $tokenString = CRM_Utils_Token::replaceHookTokens($tokenString, $contact[$contactId], $categories);
1424       }
1425     }
1426   }
1427
1428   /**
1429    * At this point, $contactDetails has loaded the contact from the DAO. Any
1430    * (non-custom) missing fields are null.  By removing them, we can avoid
1431    * expensive calls to CRM_Contact_BAO_Query.
1432    *
1433    * @param string $tokenString
1434    * @param array $contactDetails
1435    */
1436   private static function removeNullContactTokens(&$tokenString, $contactDetails, &$greetingTokens) {
1437     $greetingTokensOriginal = $greetingTokens;
1438     $contactFieldList = CRM_Contact_DAO_Contact::fields();
1439     // Sometimes contactDetails are in a multidemensional array, sometimes a
1440     // single-dimension array.
1441     if (array_key_exists(0, $contactDetails) && is_array($contactDetails[0])) {
1442       $contactDetails = current($contactDetails[0]);
1443     }
1444     $nullFields = array_keys(array_diff_key($contactFieldList, $contactDetails));
1445
1446     // Handle legacy tokens
1447     foreach (self::legacyContactTokens() as $oldToken => $newToken) {
1448       if (CRM_Utils_Array::key($newToken, $nullFields)) {
1449         $nullFields[] = $oldToken;
1450       }
1451     }
1452
1453     // Remove null contact fields from $greetingTokens
1454     $greetingTokens['contact'] = array_diff($greetingTokens['contact'], $nullFields);
1455
1456     // Also remove them from $tokenString
1457     $removedTokens = array_diff($greetingTokensOriginal['contact'], $greetingTokens['contact']);
1458     // Handle legacy tokens again, sigh
1459     if (!empty($removedTokens)) {
1460       foreach ($removedTokens as $token) {
1461         if (CRM_Utils_Array::value($token, self::legacyContactTokens()) !== NULL) {
1462           $removedTokens[] = CRM_Utils_Array::value($token, self::legacyContactTokens());
1463         }
1464       }
1465       foreach ($removedTokens as $token) {
1466         $tokenString = str_replace("{contact.$token}", '', $tokenString);
1467       }
1468     }
1469   }
1470
1471   /**
1472    * @param $tokens
1473    *
1474    * @return array
1475    */
1476   public static function flattenTokens(&$tokens) {
1477     $flattenTokens = array();
1478
1479     foreach (array(
1480                'html',
1481                'text',
1482                'subject',
1483              ) as $prop) {
1484       if (!isset($tokens[$prop])) {
1485         continue;
1486       }
1487       foreach ($tokens[$prop] as $type => $names) {
1488         if (!isset($flattenTokens[$type])) {
1489           $flattenTokens[$type] = array();
1490         }
1491         foreach ($names as $name) {
1492           $flattenTokens[$type][$name] = 1;
1493         }
1494       }
1495     }
1496
1497     return $flattenTokens;
1498   }
1499
1500   /**
1501    * Replace all user tokens in $str
1502    *
1503    * @param string $str
1504    *   The string with tokens to be replaced.
1505    *
1506    * @param null $knownTokens
1507    * @param bool $escapeSmarty
1508    *
1509    * @return string
1510    *   The processed string
1511    */
1512   public static function &replaceUserTokens($str, $knownTokens = NULL, $escapeSmarty = FALSE) {
1513     $key = 'user';
1514     if (!$knownTokens ||
1515       !isset($knownTokens[$key])
1516     ) {
1517       return $str;
1518     }
1519
1520     $str = preg_replace_callback(
1521       self::tokenRegex($key),
1522       function ($matches) use ($escapeSmarty) {
1523         return CRM_Utils_Token::getUserTokenReplacement($matches[1], $escapeSmarty);
1524       },
1525       $str
1526     );
1527     return $str;
1528   }
1529
1530   /**
1531    * @param $token
1532    * @param bool $escapeSmarty
1533    *
1534    * @return string
1535    */
1536   public static function getUserTokenReplacement($token, $escapeSmarty = FALSE) {
1537     $value = '';
1538
1539     list($objectName, $objectValue) = explode('-', $token, 2);
1540
1541     switch ($objectName) {
1542       case 'permission':
1543         $value = CRM_Core_Permission::permissionEmails($objectValue);
1544         break;
1545
1546       case 'role':
1547         $value = CRM_Core_Permission::roleEmails($objectValue);
1548         break;
1549     }
1550
1551     if ($escapeSmarty) {
1552       $value = self::tokenEscapeSmarty($value);
1553     }
1554
1555     return $value;
1556   }
1557
1558   protected static function _buildContributionTokens() {
1559     $key = 'contribution';
1560     if (self::$_tokens[$key] == NULL) {
1561       self::$_tokens[$key] = array_keys(array_merge(CRM_Contribute_BAO_Contribution::exportableFields('All'),
1562         array('campaign', 'financial_type'),
1563         self::getCustomFieldTokens('Contribution')
1564       ));
1565     }
1566   }
1567
1568   /**
1569    * Store membership tokens on the static _tokens array.
1570    */
1571   protected static function _buildMembershipTokens() {
1572     $key = 'membership';
1573     if (!isset(self::$_tokens[$key]) || self::$_tokens[$key] == NULL) {
1574       $membershipTokens = array();
1575       $tokens = CRM_Core_SelectValues::membershipTokens();
1576       foreach ($tokens as $token => $dontCare) {
1577         $membershipTokens[] = substr($token, (strpos($token, '.') + 1), -1);
1578       }
1579       self::$_tokens[$key] = $membershipTokens;
1580     }
1581   }
1582
1583   /**
1584    * Replace tokens for an entity.
1585    * @param string $entity
1586    * @param array $entityArray
1587    *   (e.g. in format from api).
1588    * @param string $str
1589    *   String to replace in.
1590    * @param array $knownTokens
1591    *   Array of tokens present.
1592    * @param bool $escapeSmarty
1593    * @return string
1594    *   string with replacements made
1595    */
1596   public static function replaceEntityTokens($entity, $entityArray, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1597     if (!$knownTokens || empty($knownTokens[$entity])) {
1598       return $str;
1599     }
1600
1601     $fn = 'get' . ucfirst($entity) . 'TokenReplacement';
1602     $fn = is_callable(array('CRM_Utils_Token', $fn)) ? $fn : 'getApiTokenReplacement';
1603     // since we already know the tokens lets just use them & do str_replace which is faster & simpler than preg_replace
1604     foreach ($knownTokens[$entity] as $token) {
1605       $replacement = self::$fn($entity, $token, $entityArray);
1606       if ($escapeSmarty) {
1607         $replacement = self::tokenEscapeSmarty($replacement);
1608       }
1609       $str = str_replace('{' . $entity . '.' . $token . '}', $replacement, $str);
1610     }
1611     return preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1612   }
1613
1614   /**
1615    * @param int $caseId
1616    * @param int $str
1617    * @param array $knownTokens
1618    * @param bool $escapeSmarty
1619    * @return string
1620    * @throws \CiviCRM_API3_Exception
1621    */
1622   public static function replaceCaseTokens($caseId, $str, $knownTokens = array(), $escapeSmarty = FALSE) {
1623     if (!$knownTokens || empty($knownTokens['case'])) {
1624       return $str;
1625     }
1626     $case = civicrm_api3('case', 'getsingle', array('id' => $caseId));
1627     return self::replaceEntityTokens('case', $case, $str, $knownTokens, $escapeSmarty);
1628   }
1629
1630   /**
1631    * Generic function for formatting token replacement for an api field
1632    *
1633    * @param string $entity
1634    * @param string $token
1635    * @param array $entityArray
1636    * @return string
1637    * @throws \CiviCRM_API3_Exception
1638    */
1639   public static function getApiTokenReplacement($entity, $token, $entityArray) {
1640     if (!isset($entityArray[$token])) {
1641       return '';
1642     }
1643     $field = civicrm_api3($entity, 'getfield', array('action' => 'get', 'name' => $token, 'get_options' => 'get'));
1644     $field = $field['values'];
1645     $fieldType = CRM_Utils_Array::value('type', $field);
1646     // Boolean fields
1647     if ($fieldType == CRM_Utils_Type::T_BOOLEAN && empty($field['options'])) {
1648       $field['options'] = array(ts('No'), ts('Yes'));
1649     }
1650     // Match pseudoconstants
1651     if (!empty($field['options'])) {
1652       $ret = array();
1653       foreach ((array) $entityArray[$token] as $val) {
1654         $ret[] = $field['options'][$val];
1655       }
1656       return implode(', ', $ret);
1657     }
1658     // Format date fields
1659     elseif ($entityArray[$token] && $fieldType == CRM_Utils_Type::T_DATE) {
1660       return CRM_Utils_Date::customFormat($entityArray[$token]);
1661     }
1662     return implode(', ', (array) $entityArray[$token]);
1663   }
1664
1665   /**
1666    * Replace Contribution tokens in html.
1667    *
1668    * @param string $str
1669    * @param array $contribution
1670    * @param bool|string $html
1671    * @param string $knownTokens
1672    * @param bool|string $escapeSmarty
1673    *
1674    * @return mixed
1675    */
1676   public static function replaceContributionTokens($str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1677     $key = 'contribution';
1678     if (!$knownTokens || !CRM_Utils_Array::value($key, $knownTokens)) {
1679       return $str; //early return
1680     }
1681     self::_buildContributionTokens();
1682
1683     // here we intersect with the list of pre-configured valid tokens
1684     // so that we remove anything we do not recognize
1685     // I hope to move this step out of here soon and
1686     // then we will just iterate on a list of tokens that are passed to us
1687
1688     $str = preg_replace_callback(
1689       self::tokenRegex($key),
1690       function ($matches) use (&$contribution, $html, $escapeSmarty) {
1691         return CRM_Utils_Token::getContributionTokenReplacement($matches[1], $contribution, $html, $escapeSmarty);
1692       },
1693       $str
1694     );
1695
1696     $str = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $str);
1697     return $str;
1698   }
1699
1700   /**
1701    * We have a situation where we are rendering more than one token in each field because we are combining
1702    * tokens from more than one contribution when pdf thank you letters are grouped (CRM-14367)
1703    *
1704    * The replaceContributionToken doesn't handle receive_date correctly in this scenario because of the formatting
1705    * it applies (other tokens are OK including date fields)
1706    *
1707    * So we sort this out & then call the main function. Note that we are not escaping smarty on this fields like the main function
1708    * does - but the fields is already being formatted through a date function
1709    *
1710    * @param string $separator
1711    * @param string $str
1712    * @param array $contribution
1713    * @param bool|string $html
1714    * @param string $knownTokens
1715    * @param bool|string $escapeSmarty
1716    *
1717    * @return string
1718    */
1719   public static function replaceMultipleContributionTokens($separator, $str, &$contribution, $html = FALSE, $knownTokens = NULL, $escapeSmarty = FALSE) {
1720     if (empty($knownTokens['contribution'])) {
1721       return $str;
1722     }
1723
1724     if (in_array('receive_date', $knownTokens['contribution'])) {
1725       $formattedDates = array();
1726       $dates = explode($separator, $contribution['receive_date']);
1727       foreach ($dates as $date) {
1728         $formattedDates[] = CRM_Utils_Date::customFormat($date, NULL, array('j', 'm', 'Y'));
1729       }
1730       $str = str_replace("{contribution.receive_date}", implode($separator, $formattedDates), $str);
1731       unset($knownTokens['contribution']['receive_date']);
1732     }
1733     return self::replaceContributionTokens($str, $contribution, $html, $knownTokens, $escapeSmarty);
1734   }
1735
1736   /**
1737    * Get replacement strings for any membership tokens (only a small number of tokens are implemnted in the first instance
1738    * - this is used by the pdfLetter task from membership search
1739    * @param string $entity
1740    *   should always be "membership"
1741    * @param string $token
1742    *   field name
1743    * @param array $membership
1744    *   An api result array for a single membership.
1745    * @return string token replacement
1746    */
1747   public static function getMembershipTokenReplacement($entity, $token, $membership) {
1748     self::_buildMembershipTokens();
1749     switch ($token) {
1750       case 'type':
1751         $value = $membership['membership_name'];
1752         break;
1753
1754       case 'status':
1755         $statuses = CRM_Member_BAO_Membership::buildOptions('status_id');
1756         $value = $statuses[$membership['status_id']];
1757         break;
1758
1759       case 'fee':
1760         try {
1761           $value = civicrm_api3('membership_type', 'getvalue', array(
1762               'id' => $membership['membership_type_id'],
1763               'return' => 'minimum_fee',
1764             ));
1765           $value = CRM_Utils_Money::format($value, NULL, NULL, TRUE);
1766         }
1767         catch (CiviCRM_API3_Exception $e) {
1768           // we can anticipate we will get an error if the minimum fee is set to 'NULL' because of the way the
1769           // api handles NULL (4.4)
1770           $value = 0;
1771         }
1772         break;
1773
1774       default:
1775         if (in_array($token, self::$_tokens[$entity])) {
1776           $value = $membership[$token];
1777           if (CRM_Utils_String::endsWith($token, '_date')) {
1778             $value = CRM_Utils_Date::customFormat($value);
1779           }
1780         }
1781         else {
1782           // ie unchanged
1783           $value = "{$entity}.{$token}";
1784         }
1785         break;
1786     }
1787
1788     return $value;
1789   }
1790
1791   /**
1792    * @param $token
1793    * @param $contribution
1794    * @param bool $html
1795    * @param bool $escapeSmarty
1796    *
1797    * @return mixed|string
1798    */
1799   public static function getContributionTokenReplacement($token, &$contribution, $html = FALSE, $escapeSmarty = FALSE) {
1800     self::_buildContributionTokens();
1801
1802     switch ($token) {
1803       case 'total_amount':
1804       case 'net_amount':
1805       case 'fee_amount':
1806       case 'non_deductible_amount':
1807         $value = CRM_Utils_Money::format(CRM_Utils_Array::retrieveValueRecursive($contribution, $token));
1808         break;
1809
1810       case 'receive_date':
1811         $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1812         $value = CRM_Utils_Date::customFormat($value, NULL, array('j', 'm', 'Y'));
1813         break;
1814
1815       default:
1816         if (!in_array($token, self::$_tokens['contribution'])) {
1817           $value = "{contribution.$token}";
1818         }
1819         else {
1820           $value = CRM_Utils_Array::retrieveValueRecursive($contribution, $token);
1821         }
1822         break;
1823     }
1824
1825     if ($escapeSmarty) {
1826       $value = self::tokenEscapeSmarty($value);
1827     }
1828     return $value;
1829   }
1830
1831   /**
1832    * @return array
1833    *   [legacy_token => new_token]
1834    */
1835   public static function legacyContactTokens() {
1836     return array(
1837       'individual_prefix' => 'prefix_id',
1838       'individual_suffix' => 'suffix_id',
1839       'gender' => 'gender_id',
1840       'communication_style' => 'communication_style_id',
1841     );
1842   }
1843
1844   /**
1845    * Get all custom field tokens of $entity
1846    *
1847    * @param string $entity
1848    * @param bool $usedForTokenWidget
1849    *
1850    * @return array $customTokens
1851    *   return custom field tokens in array('custom_N' => 'label') format
1852    */
1853   public static function getCustomFieldTokens($entity, $usedForTokenWidget = FALSE) {
1854     $customTokens = array();
1855     $tokenName = $usedForTokenWidget ? "{contribution.custom_%d}" : "custom_%d";
1856     foreach (CRM_Core_BAO_CustomField::getFields($entity) as $id => $info) {
1857       $customTokens[sprintf($tokenName, $id)] = $info['label'];
1858     }
1859
1860     return $customTokens;
1861   }
1862
1863   /**
1864    * Formats a token list for the select2 widget
1865    *
1866    * @param $tokens
1867    * @return array
1868    */
1869   public static function formatTokensForDisplay($tokens) {
1870     $sorted = $output = array();
1871
1872     // Sort in ascending order by ignoring word case
1873     natcasesort($tokens);
1874
1875     // Attempt to place tokens into optgroups
1876     // @todo These groupings could be better and less hackish. Getting them pre-grouped from upstream would be nice.
1877     foreach ($tokens as $k => $v) {
1878       // Check to see if this token is already in a group e.g. for custom fields
1879       $split = explode(' :: ', $v);
1880       if (!empty($split[1])) {
1881         $sorted[$split[1]][] = array('id' => $k, 'text' => $split[0]);
1882       }
1883       // Group by entity
1884       else {
1885         $split = explode('.', trim($k, '{}'));
1886         if (isset($split[1])) {
1887           $entity = array_key_exists($split[1], CRM_Core_DAO_Address::export()) ? 'Address' : ucfirst($split[0]);
1888         }
1889         else {
1890           $entity = 'Contact';
1891         }
1892         $sorted[ts($entity)][] = array('id' => $k, 'text' => $v);
1893       }
1894     }
1895
1896     ksort($sorted);
1897     foreach ($sorted as $k => $v) {
1898       $output[] = array('text' => $k, 'children' => $v);
1899     }
1900
1901     return $output;
1902   }
1903
1904 }