From f945f0c4920f8453b2282642178d07fceae50dac Mon Sep 17 00:00:00 2001 From: Ivan Date: Thu, 13 Aug 2020 16:40:01 +0100 Subject: [PATCH] CATL-1613: Add new field to the mail account form - Skip emails which do not have a Case ID or Case token --- CRM/Admin/Form/MailSettings.php | 5 +- CRM/Upgrade/Incremental/php/FiveThirtyOne.php | 2 + CRM/Utils/Hook.php | 22 +++ CRM/Utils/Mail/CaseMail.php | 127 ++++++++++++++++++ CRM/Utils/Mail/EmailProcessor.php | 9 ++ templates/CRM/Admin/Form/MailSettings.tpl | 6 +- templates/CRM/Admin/Page/MailSettings.hlp | 30 +++++ .../CRM/Utils/Mail/EmailProcessorTest.php | 64 ++++++++- .../Mail/data/inbound/test_cases_email.eml | 15 +++ .../data/inbound/test_non_cases_email.eml | 15 +++ xml/schema/Core/MailSettings.xml | 11 ++ 11 files changed, 301 insertions(+), 5 deletions(-) create mode 100644 CRM/Utils/Mail/CaseMail.php create mode 100644 templates/CRM/Admin/Page/MailSettings.hlp create mode 100644 tests/phpunit/CRM/Utils/Mail/data/inbound/test_cases_email.eml create mode 100644 tests/phpunit/CRM/Utils/Mail/data/inbound/test_non_cases_email.eml diff --git a/CRM/Admin/Form/MailSettings.php b/CRM/Admin/Form/MailSettings.php index 3ec4e11d0a..70c227b1ed 100644 --- a/CRM/Admin/Form/MailSettings.php +++ b/CRM/Admin/Form/MailSettings.php @@ -71,6 +71,7 @@ class CRM_Admin_Form_MailSettings extends CRM_Admin_Form { $this->add('select', 'is_default', ts('Used For?'), $usedfor); $this->addField('activity_status', ['placeholder' => FALSE]); + $this->add('checkbox', 'is_non_case_email_skipped', ts('Skip emails which do not have a Case ID or Case hash')); $this->add('checkbox', 'is_contact_creation_disabled_if_no_match', ts('Do not create new contacts when filing emails')); } @@ -148,6 +149,7 @@ class CRM_Admin_Form_MailSettings extends CRM_Admin_Form { 'is_ssl', 'is_default', 'activity_status', + 'is_non_case_email_skipped', 'is_contact_creation_disabled_if_no_match', ]; @@ -156,6 +158,7 @@ class CRM_Admin_Form_MailSettings extends CRM_Admin_Form { if (in_array($f, [ 'is_default', 'is_ssl', + 'is_non_case_email_skipped', 'is_contact_creation_disabled_if_no_match', ])) { $params[$f] = CRM_Utils_Array::value($f, $formValues, FALSE); @@ -168,7 +171,7 @@ class CRM_Admin_Form_MailSettings extends CRM_Admin_Form { $params['domain_id'] = CRM_Core_Config::domainID(); // assign id only in update mode - $status = ts('Your New Email Settings have been saved.'); + $status = ts('Your New Email Settings have been saved.'); if ($this->_action & CRM_Core_Action::UPDATE) { $params['id'] = $this->_id; $status = ts('Your Email Settings have been updated.'); diff --git a/CRM/Upgrade/Incremental/php/FiveThirtyOne.php b/CRM/Upgrade/Incremental/php/FiveThirtyOne.php index 52e7374b4c..9b5a859dc0 100644 --- a/CRM/Upgrade/Incremental/php/FiveThirtyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveThirtyOne.php @@ -74,6 +74,8 @@ class CRM_Upgrade_Incremental_php_FiveThirtyOne extends CRM_Upgrade_Incremental_ */ public function upgrade_5_31_0($rev) { $this->addTask(ts('Upgrade DB to %1: SQL', [1 => $rev]), 'runSql', $rev); + $this->addTask('Add is_non_case_email_skipped column to civicrm_mail_settings', 'addColumn', + 'civicrm_mail_settings', 'is_non_case_email_skipped', "TINYINT DEFAULT 0 NOT NULL COMMENT 'Skip emails which do not have a Case ID or Case hash'"); $this->addTask('Add is_contact_creation_disabled_if_no_match column to civicrm_mail_settings', 'addColumn', 'civicrm_mail_settings', 'is_contact_creation_disabled_if_no_match', "TINYINT DEFAULT 0 NOT NULL COMMENT 'If this option is enabled, CiviCRM will not create new contacts when filing emails'"); } diff --git a/CRM/Utils/Hook.php b/CRM/Utils/Hook.php index 7a79fdbf0b..a987971774 100644 --- a/CRM/Utils/Hook.php +++ b/CRM/Utils/Hook.php @@ -1270,6 +1270,28 @@ abstract class CRM_Utils_Hook { ->invoke(['caseTypes'], $caseTypes, self::$_nullObject, self::$_nullObject, self::$_nullObject, self::$_nullObject, self::$_nullObject, 'civicrm_caseTypes'); } + /** + * This hook is called when getting case email subject patterns. + * + * All emails related to cases have case hash/id in the subject, e.g: + * [case #ab12efg] Magic moment + * [case #1234] Magic is here + * + * Using this hook you can replace/enrich default list with some other + * patterns, e.g. include case type categories (see CiviCase extension) like: + * [(case|project|policy initiative) #hash] + * [(case|project|policy initiative) #id] + * + * @param array $subjectPatterns + * Cases related email subject regexp patterns. + * + * @return mixed + */ + public static function caseEmailSubjectPatterns(&$subjectPatterns) { + return self::singleton() + ->invoke(['caseEmailSubjectPatterns'], $subjectPatterns, self::$_nullObject, self::$_nullObject, self::$_nullObject, self::$_nullObject, self::$_nullObject, 'civicrm_caseEmailSubjectPatterns'); + } + /** * This hook is called soon after the CRM_Core_Config object has ben initialized. * You can use this hook to modify the config object and hence behavior of CiviCRM dynamically. diff --git a/CRM/Utils/Mail/CaseMail.php b/CRM/Utils/Mail/CaseMail.php new file mode 100644 index 0000000000..8b644ea658 --- /dev/null +++ b/CRM/Utils/Mail/CaseMail.php @@ -0,0 +1,127 @@ +subjectPatterns = [ + '/\[' . $this->caseLabel . ' #([0-9a-f]{7})\]/i', + '/\[' . $this->caseLabel . ' #(\d+)\]/i', + ]; + } + + /** + * Checks if email is related to cases. + * + * @param string $subject + * Email subject. + * + * @return bool + * TRUE if email subject contains case ID or case hash, FALSE otherwise. + */ + public function isCaseEmail ($subject) { + $subject = trim($subject); + $patterns = $this->getSubjectPatterns(); + $res = FALSE; + + for ($i = 0; !$res && $i < count($patterns); $i++) { + $res = preg_match($patterns[$i], $subject) === 1; + } + + return $res; + } + + /** + * Returns cases related email subject patterns. + * + * These patterns could be used to check if email is related to cases. + * + * @return array|string[] + */ + public function getSubjectPatterns() { + // Allow others to change patterns using hook. + if (empty($this->subjectPatternsHooked)) { + $patterns = $this->subjectPatterns; + CRM_Utils_Hook::caseEmailSubjectPatterns($patterns); + $this->subjectPatternsHooked = $patterns; + } + + return !empty($this->subjectPatternsHooked) + ? $this->subjectPatternsHooked + : $this->subjectPatterns; + } + + /** + * Returns value of some class property. + * + * @param string $name + * Property name. + * + * @return mixed|null + * Property value or null if property does not exist. + */ + public function get($name) { + return $this->{$name} ?? NULL; + } + + /** + * Sets value of some class property. + * + * @param string $name + * Property name. + * @param mixed $value + * New property value. + */ + public function set($name, $value) { + if (isset($this->{$name})) { + $this->{$name} = $value; + } + } + +} diff --git a/CRM/Utils/Mail/EmailProcessor.php b/CRM/Utils/Mail/EmailProcessor.php index b1dcd2a6ff..fe24b02e02 100644 --- a/CRM/Utils/Mail/EmailProcessor.php +++ b/CRM/Utils/Mail/EmailProcessor.php @@ -224,6 +224,15 @@ class CRM_Utils_Mail_EmailProcessor { // preseve backward compatibility if ($usedfor == 0 || $is_create_activities) { + // Mail account may have 'Skip emails which do not have a Case ID + // or Case hash' option, if its enabled and email is not related + // to cases - then we need to put email to ignored folder. + $caseMailUtils = new CRM_Utils_Mail_CaseMail(); + if (!empty($dao->is_non_case_email_skipped) && !$caseMailUtils->isCaseEmail($mail->subject)) { + $store->markIgnored($key); + continue; + } + // if its the activities that needs to be processed .. try { $createContact = !($dao->is_contact_creation_disabled_if_no_match ?? FALSE); diff --git a/templates/CRM/Admin/Form/MailSettings.tpl b/templates/CRM/Admin/Form/MailSettings.tpl index 9185c54a18..d003ef89f9 100644 --- a/templates/CRM/Admin/Form/MailSettings.tpl +++ b/templates/CRM/Admin/Form/MailSettings.tpl @@ -52,7 +52,9 @@ {$form.is_default.label}{$form.is_default.html}  {ts}How this mail account will be used. Only one box may be used for bounce processing. It will also be used as the envelope email when sending mass mailings.{/ts} -  {$form.is_contact_creation_disabled_if_no_match.html}{$form.is_contact_creation_disabled_if_no_match.label} +  {$form.is_non_case_email_skipped.html}{$form.is_non_case_email_skipped.label} {help id='is_non_case_email_skipped'} + +  {$form.is_contact_creation_disabled_if_no_match.html}{$form.is_contact_creation_disabled_if_no_match.label} {help id='is_contact_creation_disabled_if_no_match'}  {$form.activity_status.label}
{$form.activity_status.html}
@@ -68,9 +70,9 @@ function showActivityFields() { var fields = [ '.crm-mail-settings-form-block-activity_status', + '.crm-mail-settings-form-block-is_non_case_email_skipped', '.crm-mail-settings-form-block-is_contact_creation_disabled_if_no_match', ]; - $(fields.join(', '), $form).toggle($(this).val() === '0'); } $('select[name="is_default"]').each(showActivityFields).change(showActivityFields); diff --git a/templates/CRM/Admin/Page/MailSettings.hlp b/templates/CRM/Admin/Page/MailSettings.hlp new file mode 100644 index 0000000000..fac9cf4bc6 --- /dev/null +++ b/templates/CRM/Admin/Page/MailSettings.hlp @@ -0,0 +1,30 @@ +{* + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC. All rights reserved. | + | | + | This work is published under the GNU AGPLv3 license with some | + | permitted exceptions and without any warranty. For full license | + | and copyright information, see https://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} + +{htxt id="is_non_case_email_skipped-title"} + {ts}Skip emails which do not have a Case ID or Case hash{/ts} +{/htxt} + +{htxt id="is_non_case_email_skipped"} +

{ts}CiviCRM has functionality to file emails which contain the Case ID or Case Hash in the subject line in the format [case #1234] against a case record.{/ts}

+

{ts}Where the Case ID or Case Hash is not included CiviCRM will file the email against the contact record, by matching the email addresses on the email with any email addresses of Contact records in CiviCRM.{/ts}

+

{ts}Enabling this option will have CiviCRM skip any emails that do not have the Case ID or Case Hash so that the system will only process emails that can be placed on case records.{/ts}

+

{ts}Any emails that are not processed will be moved to the ignored folder.{/ts}

+

{ts}If email is skipped, no activities or contacts ("from"/"to"/"cc"/"bcc") will be created.{/ts}

+{/htxt} + +{htxt id="is_contact_creation_disabled_if_no_match-title"} + {ts}Do not create new contacts when filing emails{/ts} +{/htxt} + +{htxt id="is_contact_creation_disabled_if_no_match"} +

{ts}If this option is enabled, CiviCRM will not create new contacts ("from"/"to"/"cc"/"bcc") when filing emails.{/ts}

+

{ts}If the email subject contains a valid Case ID or Case hash, the email will be filed against the case.{/ts}

+{/htxt} diff --git a/tests/phpunit/CRM/Utils/Mail/EmailProcessorTest.php b/tests/phpunit/CRM/Utils/Mail/EmailProcessorTest.php index 37efac4169..b675125b75 100644 --- a/tests/phpunit/CRM/Utils/Mail/EmailProcessorTest.php +++ b/tests/phpunit/CRM/Utils/Mail/EmailProcessorTest.php @@ -37,7 +37,19 @@ class CRM_Utils_Mail_EmailProcessorTest extends CiviUnitTestCase { public function tearDown() { CRM_Utils_File::cleanDir(__DIR__ . '/data/mail'); parent::tearDown(); - $this->quickCleanup(['civicrm_group', 'civicrm_group_contact', 'civicrm_mailing', 'civicrm_mailing_job', 'civicrm_mailing_event_bounce', 'civicrm_mailing_event_queue', 'civicrm_mailing_group', 'civicrm_mailing_recipients', 'civicrm_contact', 'civicrm_email']); + $this->quickCleanup([ + 'civicrm_group', + 'civicrm_group_contact', + 'civicrm_mailing', + 'civicrm_mailing_job', + 'civicrm_mailing_event_bounce', + 'civicrm_mailing_event_queue', + 'civicrm_mailing_group', + 'civicrm_mailing_recipients', + 'civicrm_contact', + 'civicrm_email', + 'civicrm_activity', + ]); } /** @@ -139,7 +151,6 @@ class CRM_Utils_Mail_EmailProcessorTest extends CiviUnitTestCase { } /** - * * Wrapper to check for mailing bounces. * * Normally we would call $this->callAPISuccessGetCount but there is not one & there is resistance to @@ -169,6 +180,55 @@ class CRM_Utils_Mail_EmailProcessorTest extends CiviUnitTestCase { $this->eventQueue = $this->callAPISuccess('MailingEventQueue', 'get', ['api.MailingEventQueue.create' => ['hash' => 'aaaaaaaaaaaaaaaa']]); } + /** + * Set up mail account with 'Skip emails which do not have a Case ID or + * Case hash' option enabled. + */ + public function setUpSkipNonCasesEmail() { + $this->callAPISuccess('MailSettings', 'get', [ + 'api.MailSettings.create' => [ + 'name' => 'mailbox', + 'protocol' => 'Localdir', + 'source' => __DIR__ . '/data/mail', + 'domain' => 'example.com', + 'is_default' => '0', + 'is_non_case_email_skipped' => TRUE, + ], + ]); + } + + /** + * Test case email processing when is_non_case_email_skipped is enabled. + */ + public function testInboundProcessingCaseEmail() { + $this->setUpSkipNonCasesEmail(); + $mail = 'test_cases_email.eml'; + + copy(__DIR__ . '/data/inbound/' . $mail, __DIR__ . '/data/mail/' . $mail); + $this->callAPISuccess('job', 'fetch_activities', []); + $result = civicrm_api3('Activity', 'get', [ + 'sequential' => 1, + 'subject' => ['LIKE' => "%[case #214bf6d]%"], + ]); + $this->assertNotEmpty($result['values'][0]['id']); + } + + /** + * Test non case email processing when is_non_case_email_skipped is enabled. + */ + public function testInboundProcessingNonCaseEmail() { + $this->setUpSkipNonCasesEmail(); + $mail = 'test_non_cases_email.eml'; + + copy(__DIR__ . '/data/inbound/' . $mail, __DIR__ . '/data/mail/' . $mail); + $this->callAPISuccess('job', 'fetch_activities', []); + $result = civicrm_api3('Activity', 'get', [ + 'sequential' => 1, + 'subject' => ['LIKE' => "%Love letter%"], + ]); + $this->assertEmpty($result['values']); + } + /** * Set up mail account with 'Do not create new contacts when filing emails' * option enabled. diff --git a/tests/phpunit/CRM/Utils/Mail/data/inbound/test_cases_email.eml b/tests/phpunit/CRM/Utils/Mail/data/inbound/test_cases_email.eml new file mode 100644 index 0000000000..6d8cccf737 --- /dev/null +++ b/tests/phpunit/CRM/Utils/Mail/data/inbound/test_cases_email.eml @@ -0,0 +1,15 @@ +Delivered-To: to@test.test +Received: by 10.2.13.84 with SMTP id 1234567890; + Wed, 19 Dec 2018 10:01:11 +0100 (CET) +Return-Path: <> +Message-ID: +Date: Wed, 19 Dec 2018 10:01:07 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 8bit +Content-Disposition: inline +From: from@test.test +To: to@test.test +Subject: [case #214bf6d] Magic is here + +This test case is full of fun. diff --git a/tests/phpunit/CRM/Utils/Mail/data/inbound/test_non_cases_email.eml b/tests/phpunit/CRM/Utils/Mail/data/inbound/test_non_cases_email.eml new file mode 100644 index 0000000000..f5dad50c55 --- /dev/null +++ b/tests/phpunit/CRM/Utils/Mail/data/inbound/test_non_cases_email.eml @@ -0,0 +1,15 @@ +Delivered-To: to@test.test +Received: by 10.2.13.84 with SMTP id 1234567890; + Wed, 19 Dec 2018 10:01:11 +0100 (CET) +Return-Path: <> +Message-ID: +Date: Wed, 19 Dec 2018 10:01:07 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 8bit +Content-Disposition: inline +From: from@test.test +To: to@test.test +Subject: Love letter + +I love you unit test, because you are not related to cases. diff --git a/xml/schema/Core/MailSettings.xml b/xml/schema/Core/MailSettings.xml index e5f6db3107..e55f0602c3 100644 --- a/xml/schema/Core/MailSettings.xml +++ b/xml/schema/Core/MailSettings.xml @@ -150,6 +150,17 @@ Select + + is_non_case_email_skipped + Skip emails which do not have a Case ID or Case hash + boolean + 0 + + CheckBox + + Enabling this option will have CiviCRM skip any emails that do not have the Case ID or Case Hash so that the system will only process emails that can be placed on case records. Any emails that are not processed will be moved to the ignored folder. + 5.31 + is_contact_creation_disabled_if_no_match boolean -- 2.25.1