From e23e26ec0bfd35f16c477ec58088ab535522f340 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 18 May 2016 08:56:28 +1200 Subject: [PATCH] CRM-18539: Permit criteria for batch merging (other than group) --- CRM/Contact/Page/AJAX.php | 2 +- CRM/Core/BAO/PrevNextCache.php | 15 ++++++++--- CRM/Dedupe/Merger.php | 23 +++++++++++------ api/v3/Job.php | 2 +- tests/phpunit/CRM/Dedupe/MergerTest.php | 4 +-- tests/phpunit/api/v3/JobTest.php | 33 +++++++++++++++++++++++++ 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/CRM/Contact/Page/AJAX.php b/CRM/Contact/Page/AJAX.php index 2983e97720..c4a27de65b 100644 --- a/CRM/Contact/Page/AJAX.php +++ b/CRM/Contact/Page/AJAX.php @@ -667,7 +667,7 @@ LIMIT {$offset}, {$rowCount} $contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type'); } - $cacheKeyString = "merge {$contactType}_{$rgid}_{$gid}"; + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid); $searchRows = array(); $selectorElements = array('is_selected', 'is_selected_input', 'src_image', 'src', 'src_email', 'src_street', 'src_postcode', 'dst_image', 'dst', 'dst_email', 'dst_street', 'dst_postcode', 'conflicts', 'weight', 'actions'); diff --git a/CRM/Core/BAO/PrevNextCache.php b/CRM/Core/BAO/PrevNextCache.php index a78e0f3c59..8cacb0d0fe 100644 --- a/CRM/Core/BAO/PrevNextCache.php +++ b/CRM/Core/BAO/PrevNextCache.php @@ -336,15 +336,19 @@ WHERE (pn.cacheKey $op %1 OR pn.cacheKey $op %2) } /** + * Repopulate the cache of merge prospects. + * * @param int $rgid * @param int $gid * @param NULL $cacheKeyString + * @param array $criteria + * Additional criteria to filter by. * * @return bool */ - public static function refillCache($rgid = NULL, $gid = NULL, $cacheKeyString = NULL) { + public static function refillCache($rgid = NULL, $gid = NULL, $cacheKeyString = NULL, $criteria = array()) { if (!$cacheKeyString && $rgid) { - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria); } if (!$cacheKeyString) { @@ -364,7 +368,12 @@ WHERE (pn.cacheKey $op %1 OR pn.cacheKey $op %2) $foundDupes = CRM_Dedupe_Finder::dupesInGroup($rgid, $gid); } elseif ($rgid) { - $foundDupes = CRM_Dedupe_Finder::dupes($rgid); + $contactIDs = array(); + if (!empty($criteria)) { + $contacts = civicrm_api3('Contact', 'get', array_merge(array('options' => array('limit' => 0), 'return' => 'id'), $criteria['contact'])); + $contactIDs = array_keys($contacts['values']); + } + $foundDupes = CRM_Dedupe_Finder::dupes($rgid, $contactIDs); } if (!empty($foundDupes)) { diff --git a/CRM/Dedupe/Merger.php b/CRM/Dedupe/Merger.php index 7a4381848f..46ac476192 100644 --- a/CRM/Dedupe/Merger.php +++ b/CRM/Dedupe/Merger.php @@ -590,15 +590,18 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m * @param int $batchLimit number of merges to carry out in one batch. * @param int $isSelected if records with is_selected column needs to be processed. * + * @param array $criteria + * Criteria to use in the filter. + * * @return array|bool */ - public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $autoFlip = TRUE, $batchLimit = 1, $isSelected = 2) { + public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $autoFlip = TRUE, $batchLimit = 1, $isSelected = 2, $criteria = array()) { $redirectForPerformance = ($batchLimit > 1) ? TRUE : FALSE; $reloadCacheIfEmpty = (!$redirectForPerformance && $isSelected == 2); - $dupePairs = self::getDuplicatePairs($rgid, $gid, $reloadCacheIfEmpty, $batchLimit, $isSelected, '', ($mode == 'aggressive')); + $dupePairs = self::getDuplicatePairs($rgid, $gid, $reloadCacheIfEmpty, $batchLimit, $isSelected, '', ($mode == 'aggressive'), $criteria); $cacheParams = array( - 'cache_key_string' => self::getMergeCacheKeyString($rgid, $gid), + 'cache_key_string' => self::getMergeCacheKeyString($rgid, $gid, $criteria), // @todo stop passing these parameters in & instead calculate them in the merge function based // on the 'real' params like $isRespectExclusions $batchLimit and $isSelected. 'join' => self::getJoinOnDedupeTable(), @@ -1984,20 +1987,22 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m * @param bool $isSelected * @param array $orderByClause * @param bool $includeConflicts + * @param array $criteria + * Additional criteria to narrow down the merge group. * * @return array * Array of matches meeting the criteria. */ - public static function getDuplicatePairs($rule_group_id, $group_id, $reloadCacheIfEmpty, $batchLimit, $isSelected, $orderByClause = '', $includeConflicts = TRUE) { + public static function getDuplicatePairs($rule_group_id, $group_id, $reloadCacheIfEmpty, $batchLimit, $isSelected, $orderByClause = '', $includeConflicts = TRUE, $criteria = array()) { $where = self::getWhereString($batchLimit, $isSelected); - $cacheKeyString = self::getMergeCacheKeyString($rule_group_id, $group_id, $includeConflicts); + $cacheKeyString = self::getMergeCacheKeyString($rule_group_id, $group_id, $criteria); $join = self::getJoinOnDedupeTable(); $dupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where, 0, 0, array(), $orderByClause, $includeConflicts); if (empty($dupePairs) && $reloadCacheIfEmpty) { // If we haven't found any dupes, probably cache is empty. // Try filling cache and give another try. We don't need to specify include conflicts here are there will not be any // until we have done some processing. - CRM_Core_BAO_PrevNextCache::refillCache($rule_group_id, $group_id, $cacheKeyString); + CRM_Core_BAO_PrevNextCache::refillCache($rule_group_id, $group_id, $cacheKeyString, $criteria); $dupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where, 0, 0, array(), $orderByClause, $includeConflicts); return $dupePairs; } @@ -2009,14 +2014,18 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m * * @param int $rule_group_id * @param int $group_id + * @param array $criteria + * Additional criteria to narrow down the merge group. + * Currently we are only supporting the key 'contact' within it. * * @return string */ - public static function getMergeCacheKeyString($rule_group_id, $group_id) { + public static function getMergeCacheKeyString($rule_group_id, $group_id, $criteria = array()) { $contactType = CRM_Dedupe_BAO_RuleGroup::getContactTypeForRuleGroup($rule_group_id); $cacheKeyString = "merge {$contactType}"; $cacheKeyString .= $rule_group_id ? "_{$rule_group_id}" : '_0'; $cacheKeyString .= $group_id ? "_{$group_id}" : '_0'; + $cacheKeyString .= !empty($criteria) ? serialize($criteria) : '_0'; return $cacheKeyString; } diff --git a/api/v3/Job.php b/api/v3/Job.php index 33605856c2..38a08fb65b 100644 --- a/api/v3/Job.php +++ b/api/v3/Job.php @@ -490,7 +490,7 @@ function civicrm_api3_job_process_batch_merge($params) { $mode = CRM_Utils_Array::value('mode', $params, 'safe'); $autoFlip = CRM_Utils_Array::value('auto_flip', $params, TRUE); - $result = CRM_Dedupe_Merger::batchMerge($rule_group_id, $gid, $mode, $autoFlip); + $result = CRM_Dedupe_Merger::batchMerge($rule_group_id, $gid, $mode, $autoFlip, 1, 2, CRM_Utils_Array::value('criteria', $params, array())); return civicrm_api3_create_success($result, $params); } diff --git a/tests/phpunit/CRM/Dedupe/MergerTest.php b/tests/phpunit/CRM/Dedupe/MergerTest.php index 73b9bab411..4a5ecc0d2a 100644 --- a/tests/phpunit/CRM/Dedupe/MergerTest.php +++ b/tests/phpunit/CRM/Dedupe/MergerTest.php @@ -163,7 +163,7 @@ class CRM_Dedupe_MergerTest extends CiviUnitTestCase { // Retrieve pairs from prev next cache table $select = array('pn.is_selected' => 'is_selected'); - $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}"; + $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}_0"; $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.'); @@ -226,7 +226,7 @@ class CRM_Dedupe_MergerTest extends CiviUnitTestCase { // Retrieve pairs from prev next cache table $select = array('pn.is_selected' => 'is_selected'); - $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}"; + $cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}_0"; $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.'); diff --git a/tests/phpunit/api/v3/JobTest.php b/tests/phpunit/api/v3/JobTest.php index ddc56e97fd..63de713203 100644 --- a/tests/phpunit/api/v3/JobTest.php +++ b/tests/phpunit/api/v3/JobTest.php @@ -353,6 +353,39 @@ class api_v3_JobTest extends CiviUnitTestCase { ), 4); } + /** + * Test the batch merge by id range. + * + * We have 2 sets of 5 matches & set the merge only to merge the lower set. + */ + public function testBatchMergeIDRange() { + for ($x = 0; $x <= 4; $x++) { + $id = $this->individualCreate(array('email' => 'batman@gotham.met')); + } + for ($x = 0; $x <= 4; $x++) { + $this->individualCreate(array('email' => 'robin@gotham.met')); + } + $result = $this->callAPISuccess('Job', 'process_batch_merge', array('criteria' => array('contact' => array('id' => array('<' => $id))))); + $this->assertEquals(4, count($result['values']['merged'])); + $this->callAPISuccessGetCount('Contact', array('email' => 'batman@gotham.met'), 1); + $this->callAPISuccessGetCount('Contact', array('email' => 'robin@gotham.met'), 5); + $contacts = $this->callAPISuccess('Contact', 'get', array('is_deleted' => 0)); + $deletedContacts = $this->callAPISuccess('Contact', 'get', array('is_deleted' => 0)); + $this->callAPISuccessGetCount('Email', array( + 'email' => 'batman@gotham.met', + 'contact_id' => array('IN' => array_keys($contacts['values'])), + ), 1); + $this->callAPISuccessGetCount('Email', array( + 'email' => 'batman@gotham.met', + 'contact_id' => array('IN' => array_keys($deletedContacts['values'])), + ), 1); + $this->callAPISuccessGetCount('Email', array( + 'email' => 'robin@gotham.met', + 'contact_id' => array('IN' => array_keys($contacts['values'])), + ), 5); + + } + /** * Get data for batch merge. */ -- 2.25.1