tests/phpunit/api/v3/JobProcessMailingTest.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.7                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2017                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  * File for the CiviCRM APIv3 job functions
  30  *
  31  * @package CiviCRM_APIv3
  32  * @subpackage API_Job
  33  *
  34  * @copyright CiviCRM LLC (c) 2004-2017
  35  * @version $Id: Job.php 30879 2010-11-22 15:45:55Z shot $
  36  *
  37  */
  38
  39 /**
  40  * Class api_v3_JobTest
  41  * @group headless
  42  * @group civimail
  43  */
  44 class api_v3_JobProcessMailingTest extends CiviUnitTestCase {
  45   protected $_apiversion = 3;
  46
  47   public $DBResetRequired = FALSE;
  48   public $_entity = 'Job';
  49   public $_params = array();
  50   private $_groupID;
  51   private $_email;
  52
  53   protected $defaultSettings;
  54
  55   /**
  56    * @var CiviMailUtils
  57    */
  58   private $_mut;
  59
  60   public function setUp() {
  61     $this->cleanupMailingTest();
  62     parent::setUp();
  63     CRM_Mailing_BAO_MailingJob::$mailsProcessed = 0; // DGW
  64     $this->_groupID = $this->groupCreate();
  65     $this->_email = 'test@test.test';
  66     $this->_params = array(
  67       'subject' => 'Accidents in cars cause children',
  68       'body_text' => 'BEWARE children need regular infusions of toys. Santa knows your {domain.address}. There is no {action.optOutUrl}.',
  69       'name' => 'mailing name',
  70       'created_id' => 1,
  71       'groups' => array('include' => array($this->_groupID)),
  72       'scheduled_date' => 'now',
  73     );
  74     $this->defaultSettings = array(
  75       'mailings' => 1, // int, #mailings to send
  76       'recipients' => 20, // int, #contacts to receive mailing
  77       'workers' => 1, // int, #concurrent cron jobs
  78       'iterations' => 1, // int, #times to spawn all the workers
  79       'lockHold' => 0, // int, #extra seconds each cron job should hold lock
  80       'mailerBatchLimit' => 0, // int, max# recipients to send in a given cron run
  81       'mailerJobsMax' => 0, // int, max# concurrent jobs
  82       'mailerJobSize' => 0, // int, max# recipients in each job
  83       'mailThrottleTime' => 0, // int, microseconds separating messages
  84     );
  85     $this->_mut = new CiviMailUtils($this, TRUE);
  86     $this->callAPISuccess('mail_settings', 'get', array('api.mail_settings.create' => array('domain' => 'chaos.org')));
  87   }
  88
  89   /**
  90    */
  91   public function tearDown() {
  92     //$this->_mut->clearMessages();
  93     $this->_mut->stop();
  94     CRM_Utils_Hook::singleton()->reset();
  95     CRM_Mailing_BAO_MailingJob::$mailsProcessed = 0; // DGW
  96     //$this->cleanupMailingTest();
  97     parent::tearDown();
  98   }
  99
 100   public function testBasic() {
 101     $this->createContactsInGroup(10, $this->_groupID);
 102     Civi::settings()->add(array(
 103       'mailerBatchLimit' => 2,
 104     ));
 105     $this->callAPISuccess('mailing', 'create', $this->_params);
 106     $this->_mut->assertRecipients(array());
 107     $this->callAPISuccess('job', 'process_mailing', array());
 108     $this->_mut->assertRecipients($this->getRecipients(1, 2));
 109   }
 110
 111   public function concurrencyExamples() {
 112     $es = array();
 113
 114     // Launch 3 workers, but mailerJobsMax limits us to 1 worker.
 115     $es[0] = array(
 116       array(
 117         'recipients' => 20,
 118         'workers' => 3,
 119         // FIXME: lockHold is unrealistic/unrepresentative. In reality, this situation fails because
 120         // the data.* locks trample the worker.* locks. However, setting lockHold allows us to
 121         // approximate the behavior of what would happen *if* the lock-implementation didn't suffer
 122         // trampling effects.
 123         'lockHold' => 10,
 124         'mailerBatchLimit' => 4,
 125         'mailerJobsMax' => 1,
 126       ),
 127       array(
 128         0 => 2, // 2 jobs which produce 0 messages
 129         4 => 1, // 1 job which produces 4 messages
 130       ),
 131       4,
 132     );
 133
 134     // Launch 3 workers, but mailerJobsMax limits us to 2 workers.
 135     $es[1] = array(
 136       array(// Settings.
 137         'recipients' => 20,
 138         'workers' => 3,
 139         // FIXME: lockHold is unrealistic/unrepresentative. In reality, this situation fails because
 140         // the data.* locks trample the worker.* locks. However, setting lockHold allows us to
 141         // approximate the behavior of what would happen *if* the lock-implementation didn't suffer
 142         // trampling effects.
 143         'lockHold' => 10,
 144         'mailerBatchLimit' => 5,
 145         'mailerJobsMax' => 2,
 146       ),
 147       array(// Tallies.
 148         0 => 1, // 1 job which produce 0 messages
 149         5 => 2, // 2 jobs which produce 5 messages
 150       ),
 151       10, // Total sent.
 152     );
 153
 154     // Launch 3 workers and saturate them (mailerJobsMax=3)
 155     $es[2] = array(
 156       array(// Settings.
 157         'recipients' => 20,
 158         'workers' => 3,
 159         'mailerBatchLimit' => 6,
 160         'mailerJobsMax' => 3,
 161       ),
 162       array(// Tallies.
 163         6 => 3, // 3 jobs which produce 6 messages
 164       ),
 165       18, // Total sent.
 166     );
 167
 168     // Launch 4 workers and saturate them (mailerJobsMax=0)
 169     $es[3] = array(
 170       array(// Settings.
 171         'recipients' => 20,
 172         'workers' => 4,
 173         'mailerBatchLimit' => 6,
 174         'mailerJobsMax' => 0,
 175       ),
 176       array(// Tallies.
 177         6 => 3, // 3 jobs which produce 6 messages
 178         2 => 1, // 1 job which produces 2 messages
 179       ),
 180       20, // Total sent.
 181     );
 182
 183     // Launch 1 worker, 3 times in a row. Deliver everything.
 184     $es[4] = array(
 185       array(// Settings.
 186         'recipients' => 10,
 187         'workers' => 1,
 188         'iterations' => 3,
 189         'mailerBatchLimit' => 7,
 190       ),
 191       array(// Tallies.
 192         7 => 1, // 1 job which produces 7 messages
 193         3 => 1, // 1 job which produces 3 messages
 194         0 => 1, // 1 job which produces 0 messages
 195       ),
 196       10, // Total sent.
 197     );
 198
 199     // Launch 2 worker, 3 times in a row. Deliver everything.
 200     $es[5] = array(
 201       array(// Settings.
 202         'recipients' => 10,
 203         'workers' => 2,
 204         'iterations' => 3,
 205         'mailerBatchLimit' => 3,
 206       ),
 207       array(// Tallies.
 208         3 => 3, // 3 jobs which produce 3 messages
 209         1 => 1, // 1 job which produces 1 messages
 210         0 => 2, // 2 jobs which produce 0 messages
 211       ),
 212       10, // Total sent.
 213     );
 214
 215     // For two mailings, launch 1 worker, 5 times in a row. Deliver everything.
 216     $es[6] = array(
 217       array(// Settings.
 218         'mailings' => 2,
 219         'recipients' => 10,
 220         'workers' => 1,
 221         'iterations' => 5,
 222         'mailerBatchLimit' => 6,
 223       ),
 224       array(// Tallies.
 225         // x6 => x4+x2 => x6 => x2 => x0
 226         6 => 3, // 3 jobs which produce 6 messages
 227         2 => 1, // 1 job which produces 2 messages
 228         0 => 1, // 1 job which produces 0 messages
 229       ),
 230       20, // Total sent.
 231     );
 232
 233     return $es;
 234   }
 235
 236   /**
 237    * Setup various mail configuration options (eg $mailerBatchLimit,
 238    * $mailerJobMax) and spawn multiple worker threads ($workers).
 239    * Allow the threads to complete. (Optionally, repeat the above
 240    * process.) Finally, check to see if the right number of
 241    * jobs delivered the right number of messages.
 242    *
 243    * @param array $settings
 244    *   An array of settings (eg mailerBatchLimit, workers). See comments
 245    *   for $this->defaultSettings.
 246    * @param array $expectedTallies
 247    *    A listing of the number cron-runs keyed by their size.
 248    *    For example, array(10=>2) means that there 2 cron-runs
 249    *    which delivered 10 messages each.
 250    * @param int $expectedTotal
 251    *    The total number of contacts for whom messages should have
 252    *    been sent.
 253    * @dataProvider concurrencyExamples
 254    */
 255   public function testConcurrency($settings, $expectedTallies, $expectedTotal) {
 256     $settings = array_merge($this->defaultSettings, $settings);
 257
 258     $this->createContactsInGroup($settings['recipients'], $this->_groupID);
 259     Civi::settings()->add(CRM_Utils_Array::subset($settings, array(
 260       'mailerBatchLimit',
 261       'mailerJobsMax',
 262       'mailThrottleTime',
 263     )));
 264
 265     for ($i = 0; $i < $settings['mailings']; $i++) {
 266       $this->callAPISuccess('mailing', 'create', $this->_params);
 267     }
 268
 269     $this->_mut->assertRecipients(array());
 270
 271     $allApiResults = array();
 272     for ($iterationId = 0; $iterationId < $settings['iterations']; $iterationId++) {
 273       $apiCalls = $this->createExternalAPI();
 274       $apiCalls->addEnv(array('CIVICRM_CRON_HOLD' => $settings['lockHold']));
 275       for ($workerId = 0; $workerId < $settings['workers']; $workerId++) {
 276         $apiCalls->addCall('job', 'process_mailing', array());
 277       }
 278       $apiCalls->start();
 279       $this->assertEquals($settings['workers'], $apiCalls->getRunningCount());
 280
 281       $apiCalls->wait();
 282       $allApiResults = array_merge($allApiResults, $apiCalls->getResults());
 283     }
 284
 285     $actualTallies = $this->tallyApiResults($allApiResults);
 286     $this->assertEquals($expectedTallies, $actualTallies, 'API tallies should match.' . print_r(array(
 287         'expectedTallies' => $expectedTallies,
 288         'actualTallies' => $actualTallies,
 289         'apiResults' => $allApiResults,
 290       ), TRUE));
 291     $this->_mut->assertRecipients($this->getRecipients(1, $expectedTotal / $settings['mailings'], 'nul.example.com', $settings['mailings']));
 292     $this->assertEquals(0, $apiCalls->getRunningCount());
 293   }
 294
 295   /**
 296    * Create contacts in group.
 297    *
 298    * @param int $count
 299    * @param int $groupID
 300    * @param string $domain
 301    */
 302   public function createContactsInGroup($count, $groupID, $domain = 'nul.example.com') {
 303     for ($i = 1; $i <= $count; $i++) {
 304       $contactID = $this->individualCreate(array('first_name' => $count, 'email' => 'mail' . $i . '@' . $domain));
 305       $this->callAPISuccess('group_contact', 'create', array(
 306         'contact_id' => $contactID,
 307         'group_id' => $groupID,
 308         'status' => 'Added',
 309       ));
 310     }
 311   }
 312
 313   /**
 314    * Construct the list of email addresses for $count recipients.
 315    *
 316    * @param int $start
 317    * @param int $count
 318    * @param string $domain
 319    * @param int $mailings
 320    *
 321    * @return array
 322    */
 323   public function getRecipients($start, $count, $domain = 'nul.example.com', $mailings = 1) {
 324     $recipients = array();
 325     for ($m = 0; $m < $mailings; $m++) {
 326       for ($i = $start; $i < ($start + $count); $i++) {
 327         $recipients[][0] = 'mail' . $i . '@' . $domain;
 328       }
 329     }
 330     return $recipients;
 331   }
 332
 333   protected function cleanupMailingTest() {
 334     $this->quickCleanup(array(
 335       'civicrm_mailing',
 336       'civicrm_mailing_job',
 337       'civicrm_mailing_spool',
 338       'civicrm_mailing_group',
 339       'civicrm_mailing_recipients',
 340       'civicrm_mailing_event_queue',
 341       'civicrm_mailing_event_bounce',
 342       'civicrm_mailing_event_delivered',
 343       'civicrm_group',
 344       'civicrm_group_contact',
 345       'civicrm_contact',
 346     ));
 347   }
 348
 349   /**
 350    * Categorize results based on (a) whether they succeeded
 351    * and (b) the number of messages sent.
 352    *
 353    * @param array $apiResults
 354    * @return array
 355    *   One key 'error' for all failures.
 356    *   A separate key for each distinct quantity.
 357    */
 358   protected function tallyApiResults($apiResults) {
 359     $ret = array();
 360     foreach ($apiResults as $apiResult) {
 361       $key = !empty($apiResult['is_error']) ? 'error' : $apiResult['values']['processed'];
 362       $ret[$key] = !empty($ret[$key]) ? 1 + $ret[$key] : 1;
 363     }
 364     return $ret;
 365   }
 366
 367 }