tests/phpunit/CRM/Dedupe/MergerTest.php

   1 <?php
   2
   3 /**
   4  * Class CRM_Dedupe_DedupeMergerTest
   5  * @group headless
   6  */
   7 class CRM_Dedupe_MergerTest extends CiviUnitTestCase {
   8
   9   protected $_groupId;
  10   protected $_contactIds = array();
  11
  12   public function tearDown() {
  13     $this->quickCleanup(array('civicrm_contact', 'civicrm_group_contact', 'civicrm_group'));
  14     parent::tearDown();
  15   }
  16
  17   public function createDupeContacts() {
  18     // create a group to hold contacts, so that dupe checks don't consider any other contacts in the DB
  19     $params = array(
  20       'name'       => 'Test Dupe Merger Group',
  21       'title'      => 'Test Dupe Merger Group',
  22       'domain_id'  => 1,
  23       'is_active'  => 1,
  24       'visibility' => 'Public Pages',
  25     );
  26
  27     $result = $this->callAPISuccess('group', 'create', $params);
  28     $this->_groupId = $result['id'];
  29
  30     // contact data set
  31
  32     // make dupe checks based on based on following contact sets:
  33     // FIRST - LAST - EMAIL
  34     // ---------------------------------
  35     // robin  - hood - robin@example.com
  36     // robin  - hood - robin@example.com
  37     // robin  - hood - hood@example.com
  38     // robin  - dale - robin@example.com
  39     // little - dale - dale@example.com
  40     // little - dale - dale@example.com
  41     // will   - dale - dale@example.com
  42     // will   - dale - will@example.com
  43     // will   - dale - will@example.com
  44     $params = array(
  45       array(
  46         'first_name' => 'robin',
  47         'last_name' => 'hood',
  48         'email' => 'robin@example.com',
  49         'contact_type' => 'Individual',
  50       ),
  51       array(
  52         'first_name' => 'robin',
  53         'last_name' => 'hood',
  54         'email' => 'robin@example.com',
  55         'contact_type' => 'Individual',
  56       ),
  57       array(
  58         'first_name' => 'robin',
  59         'last_name' => 'hood',
  60         'email' => 'hood@example.com',
  61         'contact_type' => 'Individual',
  62       ),
  63       array(
  64         'first_name' => 'robin',
  65         'last_name' => 'dale',
  66         'email' => 'robin@example.com',
  67         'contact_type' => 'Individual',
  68       ),
  69       array(
  70         'first_name' => 'little',
  71         'last_name' => 'dale',
  72         'email' => 'dale@example.com',
  73         'contact_type' => 'Individual',
  74       ),
  75       array(
  76         'first_name' => 'little',
  77         'last_name' => 'dale',
  78         'email' => 'dale@example.com',
  79         'contact_type' => 'Individual',
  80       ),
  81       array(
  82         'first_name' => 'will',
  83         'last_name' => 'dale',
  84         'email' => 'dale@example.com',
  85         'contact_type' => 'Individual',
  86       ),
  87       array(
  88         'first_name' => 'will',
  89         'last_name' => 'dale',
  90         'email' => 'will@example.com',
  91         'contact_type' => 'Individual',
  92       ),
  93       array(
  94         'first_name' => 'will',
  95         'last_name' => 'dale',
  96         'email' => 'will@example.com',
  97         'contact_type' => 'Individual',
  98       ),
  99     );
 100
 101     $count = 1;
 102     foreach ($params as $param) {
 103       $param['version'] = 3;
 104       $contact = civicrm_api('contact', 'create', $param);
 105       $this->_contactIds[$count++] = $contact['id'];
 106
 107       $grpParams = array(
 108         'contact_id' => $contact['id'],
 109         'group_id'   => $this->_groupId,
 110         'version'    => 3,
 111       );
 112       $this->callAPISuccess('group_contact', 'create', $grpParams);
 113     }
 114   }
 115
 116   /**
 117    * Delete all created contacts.
 118    */
 119   public function deleteDupeContacts() {
 120     foreach ($this->_contactIds as $contactId) {
 121       $this->contactDelete($contactId);
 122     }
 123     $this->groupDelete($this->_groupId);
 124   }
 125
 126   /**
 127    * Test the batch merge.
 128    */
 129   public function testBatchMergeSelectedDuplicates() {
 130     $this->createDupeContacts();
 131
 132     // verify that all contacts have been created separately
 133     $this->assertEquals(count($this->_contactIds), 9, 'Check for number of contacts.');
 134
 135     $dao = new CRM_Dedupe_DAO_RuleGroup();
 136     $dao->contact_type = 'Individual';
 137     $dao->name = 'IndividualSupervised';
 138     $dao->is_default = 1;
 139     $dao->find(TRUE);
 140
 141     $foundDupes = CRM_Dedupe_Finder::dupesInGroup($dao->id, $this->_groupId);
 142
 143     // -------------------------------------------------------------------------
 144     // Name and Email (reserved) Matches ( 3 pairs )
 145     // --------------------------------------------------------------------------
 146     // robin  - hood - robin@example.com
 147     // robin  - hood - robin@example.com
 148     // little - dale - dale@example.com
 149     // little - dale - dale@example.com
 150     // will   - dale - will@example.com
 151     // will   - dale - will@example.com
 152     // so 3 pairs for - first + last + mail
 153     $this->assertEquals(count($foundDupes), 3, 'Check Individual-Supervised dupe rule for dupesInGroup().');
 154
 155     // Run dedupe finder as the browser would
 156     $_SERVER['REQUEST_METHOD'] = 'GET'; //avoid invalid key error
 157     $object = new CRM_Contact_Page_DedupeFind();
 158     $object->set('gid', $this->_groupId);
 159     $object->set('rgid', $dao->id);
 160     $object->set('action', CRM_Core_Action::UPDATE);
 161     $object->setEmbedded(TRUE);
 162     @$object->run();
 163
 164     // Retrieve pairs from prev next cache table
 165     $select = array('pn.is_selected' => 'is_selected');
 166     $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}_0";
 167     $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
 168
 169     $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.');
 170
 171     // mark first two pairs as selected
 172     CRM_Core_DAO::singleValueQuery("UPDATE civicrm_prevnext_cache SET is_selected = 1 WHERE id IN ({$pnDupePairs[0]['prevnext_id']}, {$pnDupePairs[1]['prevnext_id']})");
 173
 174     $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
 175     $this->assertEquals($pnDupePairs[0]['is_selected'], 1, 'Check if first record in dupe pairs is marked as selected.');
 176     $this->assertEquals($pnDupePairs[0]['is_selected'], 1, 'Check if second record in dupe pairs is marked as selected.');
 177
 178     // batch merge selected dupes
 179     $result = CRM_Dedupe_Merger::batchMerge($dao->id, $this->_groupId, 'safe', TRUE, 5, 1);
 180     $this->assertEquals(count($result['merged']), 2, 'Check number of merged pairs.');
 181
 182     // retrieve pairs from prev next cache table
 183     $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
 184     $this->assertEquals(count($pnDupePairs), 1, 'Check number of remaining dupe pairs in prev next cache.');
 185
 186     $this->deleteDupeContacts();
 187   }
 188
 189   /**
 190    * Test the batch merge.
 191    */
 192   public function testBatchMergeAllDuplicates() {
 193     $this->createDupeContacts();
 194
 195     // verify that all contacts have been created separately
 196     $this->assertEquals(count($this->_contactIds), 9, 'Check for number of contacts.');
 197
 198     $dao = new CRM_Dedupe_DAO_RuleGroup();
 199     $dao->contact_type = 'Individual';
 200     $dao->name = 'IndividualSupervised';
 201     $dao->is_default = 1;
 202     $dao->find(TRUE);
 203
 204     $foundDupes = CRM_Dedupe_Finder::dupesInGroup($dao->id, $this->_groupId);
 205
 206     // -------------------------------------------------------------------------
 207     // Name and Email (reserved) Matches ( 3 pairs )
 208     // --------------------------------------------------------------------------
 209     // robin  - hood - robin@example.com
 210     // robin  - hood - robin@example.com
 211     // little - dale - dale@example.com
 212     // little - dale - dale@example.com
 213     // will   - dale - will@example.com
 214     // will   - dale - will@example.com
 215     // so 3 pairs for - first + last + mail
 216     $this->assertEquals(count($foundDupes), 3, 'Check Individual-Supervised dupe rule for dupesInGroup().');
 217
 218     // Run dedupe finder as the browser would
 219     $_SERVER['REQUEST_METHOD'] = 'GET'; //avoid invalid key error
 220     $object = new CRM_Contact_Page_DedupeFind();
 221     $object->set('gid', $this->_groupId);
 222     $object->set('rgid', $dao->id);
 223     $object->set('action', CRM_Core_Action::UPDATE);
 224     $object->setEmbedded(TRUE);
 225     @$object->run();
 226
 227     // Retrieve pairs from prev next cache table
 228     $select = array('pn.is_selected' => 'is_selected');
 229     $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}_0";
 230     $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
 231
 232     $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.');
 233
 234     // batch merge all dupes
 235     $result = CRM_Dedupe_Merger::batchMerge($dao->id, $this->_groupId, 'safe', TRUE, 5, 2);
 236     $this->assertEquals(count($result['merged']), 3, 'Check number of merged pairs.');
 237
 238     // retrieve pairs from prev next cache table
 239     $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
 240     $this->assertEquals(count($pnDupePairs), 0, 'Check number of remaining dupe pairs in prev next cache.');
 241
 242     $this->deleteDupeContacts();
 243   }
 244
 245   /**
 246    * The goal of this function is to test that all required tables are returned.
 247    */
 248   public function testGetCidRefs() {
 249     $this->entityCustomGroupWithSingleFieldCreate(__FUNCTION__, 'Contacts');
 250     $this->assertEquals(array_merge($this->getStaticCIDRefs(), $this->getHackedInCIDRef()), CRM_Dedupe_Merger::cidRefs());
 251     $this->assertEquals(array_merge($this->getCalculatedCIDRefs(), $this->getHackedInCIDRef()), CRM_Dedupe_Merger::cidRefs());
 252   }
 253
 254   /**
 255    * Get the list of not-really-cid-refs that are currently hacked in.
 256    *
 257    * This is hacked into getCIDs function.
 258    *
 259    * @return array
 260    */
 261   public function getHackedInCIDRef() {
 262     return array(
 263       'civicrm_entity_tag' => array(
 264         0 => 'entity_id',
 265       ),
 266     );
 267   }
 268
 269   /**
 270    * Test function that gets duplicate pairs.
 271    *
 272    * It turns out there are 2 code paths retrieving this data so my initial focus is on ensuring
 273    * they match.
 274    */
 275   public function testGetMatches() {
 276     $this->setupMatchData();
 277     $pairs = CRM_Dedupe_Merger::getDuplicatePairs(
 278       1,
 279       NULL,
 280       TRUE,
 281       25,
 282       FALSE
 283     );
 284
 285     $this->assertEquals(array(
 286       0 => array(
 287         'srcID' => $this->contacts[0]['id'],
 288         'srcName' => 'Mr. Mickey Mouse II',
 289         'dstID' => $this->contacts[1]['id'],
 290         'dstName' => 'Mr. Mickey Mouse II',
 291         'weight' => 20,
 292         'canMerge' => TRUE,
 293       ),
 294       1 => array(
 295         'srcID' => $this->contacts[2]['id'],
 296         'srcName' => 'Mr. Minnie Mouse II',
 297         'dstID' => $this->contacts[3]['id'],
 298         'dstName' => 'Mr. Minnie Mouse II',
 299         'weight' => 20,
 300         'canMerge' => TRUE,
 301       ),
 302     ), $pairs);
 303   }
 304
 305   /**
 306    * Test function that gets duplicate pairs.
 307    *
 308    * It turns out there are 2 code paths retrieving this data so my initial focus is on ensuring
 309    * they match.
 310    */
 311   public function testGetMatchesInGroup() {
 312     $this->setupMatchData();
 313
 314     $groupID = $this->groupCreate(array('title' => 'she-mice'));
 315
 316     $this->callAPISuccess('GroupContact', 'create', array('group_id' => $groupID, 'contact_id' => $this->contacts[3]['id']));
 317
 318     $pairs = CRM_Dedupe_Merger::getDuplicatePairs(
 319       1,
 320       $groupID,
 321       TRUE,
 322       25,
 323       FALSE
 324     );
 325
 326     $this->assertEquals(array(
 327       0 => array(
 328         'srcID' => $this->contacts[3]['id'],
 329         'srcName' => 'Mr. Minnie Mouse II',
 330         'dstID' => $this->contacts[2]['id'],
 331         'dstName' => 'Mr. Minnie Mouse II',
 332         'weight' => 20,
 333         'canMerge' => TRUE,
 334       ),
 335     ), $pairs);
 336   }
 337
 338   /**
 339    * Set up some contacts for our matching.
 340    */
 341   public function setupMatchData() {
 342     $fixtures = array(
 343       array(
 344         'first_name' => 'Mickey',
 345         'last_name' => 'Mouse',
 346         'email' => 'mickey@mouse.com',
 347       ),
 348       array(
 349         'first_name' => 'Mickey',
 350         'last_name' => 'Mouse',
 351         'email' => 'mickey@mouse.com',
 352       ),
 353       array(
 354         'first_name' => 'Minnie',
 355         'last_name' => 'Mouse',
 356         'email' => 'mickey@mouse.com',
 357       ),
 358       array(
 359         'first_name' => 'Minnie',
 360         'last_name' => 'Mouse',
 361         'email' => 'mickey@mouse.com',
 362       ),
 363     );
 364     foreach ($fixtures as $fixture) {
 365       $contactID = $this->individualCreate($fixture);
 366       $this->contacts[] = array_merge($fixture, array('id' => $contactID));
 367     }
 368   }
 369
 370
 371   /**
 372    * Get the list of tables that refer to the CID.
 373    *
 374    * This is a statically maintained (in this test list).
 375    *
 376    * There is also a check against an automated list but having both seems to add extra stability to me. They do
 377    * not change often.
 378    */
 379   public function getStaticCIDRefs() {
 380     return array(
 381       'civicrm_acl_cache' => array(
 382         0 => 'contact_id',
 383       ),
 384       'civicrm_acl_contact_cache' => array(
 385         0 => 'user_id',
 386         1 => 'contact_id',
 387       ),
 388       'civicrm_action_log' => array(
 389         0 => 'contact_id',
 390       ),
 391       'civicrm_activity_contact' => array(
 392         0 => 'contact_id',
 393       ),
 394       'civicrm_address' => array(
 395         0 => 'contact_id',
 396       ),
 397       'civicrm_batch' => array(
 398         0 => 'created_id',
 399         1 => 'modified_id',
 400       ),
 401       'civicrm_campaign' => array(
 402         0 => 'created_id',
 403         1 => 'last_modified_id',
 404       ),
 405       'civicrm_case_contact' => array(
 406         0 => 'contact_id',
 407       ),
 408       'civicrm_contact' => array(
 409         0 => 'primary_contact_id',
 410         1 => 'employer_id',
 411       ),
 412       'civicrm_contribution' => array(
 413         0 => 'contact_id',
 414       ),
 415       'civicrm_contribution_page' => array(
 416         0 => 'created_id',
 417       ),
 418       'civicrm_contribution_recur' => array(
 419         0 => 'contact_id',
 420       ),
 421       'civicrm_contribution_soft' => array(
 422         0 => 'contact_id',
 423       ),
 424       'civicrm_custom_group' => array(
 425         0 => 'created_id',
 426       ),
 427       'civicrm_dashboard_contact' => array(
 428         0 => 'contact_id',
 429       ),
 430       'civicrm_dedupe_exception' => array(
 431         0 => 'contact_id1',
 432         1 => 'contact_id2',
 433       ),
 434       'civicrm_domain' => array(
 435         0 => 'contact_id',
 436       ),
 437       'civicrm_email' => array(
 438         0 => 'contact_id',
 439       ),
 440       'civicrm_event' => array(
 441         0 => 'created_id',
 442       ),
 443       'civicrm_event_carts' => array(
 444         0 => 'user_id',
 445       ),
 446       'civicrm_financial_account' => array(
 447         0 => 'contact_id',
 448       ),
 449       'civicrm_financial_item' => array(
 450         0 => 'contact_id',
 451       ),
 452       'civicrm_grant' => array(
 453         0 => 'contact_id',
 454       ),
 455       'civicrm_group' => array(
 456         0 => 'created_id',
 457         1 => 'modified_id',
 458       ),
 459       'civicrm_group_contact' => array(
 460         0 => 'contact_id',
 461       ),
 462       'civicrm_group_contact_cache' => array(
 463         0 => 'contact_id',
 464       ),
 465       'civicrm_group_organization' => array(
 466         0 => 'organization_id',
 467       ),
 468       'civicrm_im' => array(
 469         0 => 'contact_id',
 470       ),
 471       'civicrm_log' => array(
 472         0 => 'modified_id',
 473       ),
 474       'civicrm_mailing' => array(
 475         0 => 'created_id',
 476         1 => 'scheduled_id',
 477         2 => 'approver_id',
 478       ),
 479       'civicrm_mailing_abtest' => array(
 480         0 => 'created_id',
 481       ),
 482       'civicrm_mailing_event_queue' => array(
 483         0 => 'contact_id',
 484       ),
 485       'civicrm_mailing_event_subscribe' => array(
 486         0 => 'contact_id',
 487       ),
 488       'civicrm_mailing_recipients' => array(
 489         0 => 'contact_id',
 490       ),
 491       'civicrm_membership' => array(
 492         0 => 'contact_id',
 493       ),
 494       'civicrm_membership_log' => array(
 495         0 => 'modified_id',
 496       ),
 497       'civicrm_membership_type' => array(
 498         0 => 'member_of_contact_id',
 499       ),
 500       'civicrm_note' => array(
 501         0 => 'contact_id',
 502       ),
 503       'civicrm_openid' => array(
 504         0 => 'contact_id',
 505       ),
 506       'civicrm_participant' => array(
 507         0 => 'contact_id',
 508         1 => 'transferred_to_contact_id', //CRM-16761
 509       ),
 510       'civicrm_payment_token' => array(
 511         0 => 'contact_id',
 512         1 => 'created_id',
 513       ),
 514       'civicrm_pcp' => array(
 515         0 => 'contact_id',
 516       ),
 517       'civicrm_phone' => array(
 518         0 => 'contact_id',
 519       ),
 520       'civicrm_pledge' => array(
 521         0 => 'contact_id',
 522       ),
 523       'civicrm_print_label' => array(
 524         0 => 'created_id',
 525       ),
 526       'civicrm_relationship' => array(
 527         0 => 'contact_id_a',
 528         1 => 'contact_id_b',
 529       ),
 530       'civicrm_report_instance' => array(
 531         0 => 'created_id',
 532         1 => 'owner_id',
 533       ),
 534       'civicrm_setting' => array(
 535         0 => 'contact_id',
 536         1 => 'created_id',
 537       ),
 538       'civicrm_subscription_history' => array(
 539         0 => 'contact_id',
 540       ),
 541       'civicrm_survey' => array(
 542         0 => 'created_id',
 543         1 => 'last_modified_id',
 544       ),
 545       'civicrm_tag' => array(
 546         0 => 'created_id',
 547       ),
 548       'civicrm_uf_group' => array(
 549         0 => 'created_id',
 550       ),
 551       'civicrm_uf_match' => array(
 552         0 => 'contact_id',
 553       ),
 554       'civicrm_value_testgetcidref_1' => array(
 555         0 => 'entity_id',
 556       ),
 557       'civicrm_website' => array(
 558         0 => 'contact_id',
 559       ),
 560     );
 561   }
 562
 563   /**
 564    * Get a list of CIDs that is calculated off the schema.
 565    *
 566    * Note this is an expensive and table locking query. Should be safe in tests though.
 567    */
 568   public function getCalculatedCIDRefs() {
 569     $cidRefs = array();
 570     $sql = "
 571 SELECT
 572     table_name,
 573     column_name
 574 FROM information_schema.key_column_usage
 575 WHERE
 576     referenced_table_schema = database() AND
 577     referenced_table_name = 'civicrm_contact' AND
 578     referenced_column_name = 'id';
 579       ";
 580     $dao = CRM_Core_DAO::executeQuery($sql);
 581     while ($dao->fetch()) {
 582       $cidRefs[$dao->table_name][] = $dao->column_name;
 583     }
 584     // Do specific re-ordering changes to make this the same as the ref validated one.
 585     // The above query orders by FK alphabetically.
 586     // There might be cleverer ways to do this but it shouldn't change much.
 587     $cidRefs['civicrm_contact'][0] = 'primary_contact_id';
 588     $cidRefs['civicrm_contact'][1] = 'employer_id';
 589     $cidRefs['civicrm_acl_contact_cache'][0] = 'user_id';
 590     $cidRefs['civicrm_acl_contact_cache'][1] = 'contact_id';
 591     $cidRefs['civicrm_mailing'][0] = 'created_id';
 592     $cidRefs['civicrm_mailing'][1] = 'scheduled_id';
 593     $cidRefs['civicrm_mailing'][2] = 'approver_id';
 594     return $cidRefs;
 595   }
 596
 597 }