From 63ef778e5b40866ad4a89140a31ccba5553ab729 Mon Sep 17 00:00:00 2001 From: deepak-srivastava Date: Thu, 14 May 2015 11:01:42 +0100 Subject: [PATCH] Applying patch from Civi v4.5 repo, dedupe-workflow-45 branch, till 97e53e0c63c9d0d4d4712787e6adcc17dcbf188d (re-order columns and disable sorting for icon column) --- CRM/Contact/Page/AJAX.php | 234 +++++++++++++++--- CRM/Contact/Page/DedupeFind.php | 25 +- CRM/Contact/Page/DedupeMerge.php | 129 ++++++++++ CRM/Core/BAO/PrevNextCache.php | 89 ++++++- CRM/Core/xml/Menu/Contact.xml | 11 + CRM/Dedupe/Merger.php | 97 +++++++- CRM/Utils/JSON.php | 33 +++ templates/CRM/Contact/Page/DedupeFind.tpl | 262 ++++++++++++++++++++- templates/CRM/Contact/Page/DedupeMerge.tpl | 25 ++ 9 files changed, 843 insertions(+), 62 deletions(-) create mode 100644 CRM/Contact/Page/DedupeMerge.php create mode 100644 templates/CRM/Contact/Page/DedupeMerge.tpl diff --git a/CRM/Contact/Page/AJAX.php b/CRM/Contact/Page/AJAX.php index a2a9b3bd9c..e597802c87 100644 --- a/CRM/Contact/Page/AJAX.php +++ b/CRM/Contact/Page/AJAX.php @@ -668,53 +668,194 @@ LIMIT {$offset}, {$rowCount} CRM_Utils_JSON::output(array('status' => ($status) ? $oper : $status)); } - public static function getDedupes() { - - $sEcho = CRM_Utils_Type::escape($_REQUEST['sEcho'], 'Integer'); - $offset = isset($_REQUEST['iDisplayStart']) ? CRM_Utils_Type::escape($_REQUEST['iDisplayStart'], 'Integer') : 0; - $rowCount = isset($_REQUEST['iDisplayLength']) ? CRM_Utils_Type::escape($_REQUEST['iDisplayLength'], 'Integer') : 25; - $sort = 'sort_name'; - $sortOrder = isset($_REQUEST['sSortDir_0']) ? CRM_Utils_Type::escape($_REQUEST['sSortDir_0'], 'String') : 'asc'; - - $gid = isset($_REQUEST['gid']) ? CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer') : 0; - $rgid = isset($_REQUEST['rgid']) ? CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer') : 0; + static function getDedupes() { + $offset = isset($_REQUEST['start']) ? CRM_Utils_Type::escape($_REQUEST['start'], 'Integer') : 0; + $rowCount = isset($_REQUEST['length']) ? CRM_Utils_Type::escape($_REQUEST['length'], 'Integer') : 25; + + $gid = isset($_REQUEST['gid']) ? CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer') : 0; + $rgid = isset($_REQUEST['rgid']) ? CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer') : 0; + $selected = isset($_REQUEST['selected']) ? CRM_Utils_Type::escape($_REQUEST['selected'], 'Integer') : 0; + if ($rowCount < 0) { + $rowCount = 0; + } $contactType = ''; if ($rgid) { $contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type'); } - $cacheKeyString = "merge {$contactType}_{$rgid}_{$gid}"; - $searchRows = array(); - $selectorElements = array('src', 'dst', 'weight', 'actions'); + $cacheKeyString = "merge {$contactType}_{$rgid}_{$gid}"; + $searchRows = array(); + $selectorElements = array('is_selected', 'is_selected_input', 'src_image', 'src', 'src_email', 'src_street', 'src_postcode', 'dst_image', 'dst', 'dst_email', 'dst_street', 'dst_postcode', 'conflicts', 'weight', 'actions'); - $join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND - pn.entity_id2 = de.contact_id2 )"; - $where = "de.id IS NULL"; + foreach ($_REQUEST['columns'] as $columnInfo) { + if (!empty($columnInfo['search']['value'])) { + ${$columnInfo['data']} = CRM_Utils_Type::escape($columnInfo['search']['value'], 'String'); + } + } + $join = ''; + $where = array(); + $searchData = CRM_Utils_Array::value('search', $_REQUEST); + if ($src || !empty($searchData['value']) ) { + $src = $src ? $src : $searchData['value']; + $where[] = " cc1.display_name LIKE '%{$src}%'"; + } + if ($dst || !empty($searchData['value'])) { + $dst = $dst ? $dst : $searchData['value']; + $where[] = " cc2.display_name LIKE '%{$dst}%'"; + } + if ($src_email || !empty($searchData['value'])) { + $src_email = $src_email ? $src_email : $searchData['value']; + $where[] = " (ce1.is_primary = 1 AND ce1.email LIKE '%{$src_email}%')"; + } + if ($dst_email || !empty($searchData['value'])) { + $dst_email = $dst_email ? $dst_email : $searchData['value']; + $where[] = " (ce2.is_primary = 1 AND ce2.email LIKE '%{$dst_email}%')"; + } + if ($src_postcode || !empty($searchData['value'])) { + $src_postcode = $src_postcode ? $src_postcode : $searchData['value']; + $where[] = " (ca1.is_primary = 1 AND ca1.postal_code LIKE '%{$src_postcode}%')"; + } + if ($dst_postcode || !empty($searchData['value'])) { + $dst_postcode = $dst_postcode ? $dst_postcode : $searchData['value']; + $where[] = " (ca2.is_primary = 1 AND ca2.postal_code LIKE '%{$dst_postcode}%')"; + } + if ($src_street || !empty($searchData['value'])) { + $src_street = $src_street ? $src_street : $searchData['value']; + $where[] = " (ca1.is_primary = 1 AND ca1.street_address LIKE '%{$src_street}%')"; + } + if ($dst_street || !empty($searchData['value'])) { + $dst_street = $dst_street ? $dst_street : $searchData['value']; + $where[] = " (ca2.is_primary = 1 AND ca2.street_address LIKE '%{$dst_street}%')"; + } + if (!empty($searchData['value'])) { + $whereClause = ' ( '.implode(' OR ', $where).' ) '; + } + else { + if (!empty($where)) { + $whereClause = implode(' AND ', $where); + } + } + $whereClause .= $whereClause ? ' AND de.id IS NULL' : ' de.id IS NULL'; - $iFilteredTotal = $iTotal = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, $join, $where); - $mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where, $offset, $rowCount); + if ($selected) { + $whereClause .= ' AND pn.is_selected = 1'; + } + $join .= " LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND pn.entity_id2 = de.contact_id2 )"; + + + $select = array( + 'cc1.contact_type' => 'src_contact_type', + 'cc1.display_name' => 'src_display_name', + 'cc1.contact_sub_type'=> 'src_contact_sub_type', + 'cc2.contact_type' => 'dst_contact_type', + 'cc2.display_name' => 'dst_display_name', + 'cc2.contact_sub_type'=> 'dst_contact_sub_type', + 'ce1.email' => 'src_email', + 'ce2.email' => 'dst_email', + 'ca1.postal_code' => 'src_postcode', + 'ca2.postal_code' => 'dst_postcode', + 'ca1.street_address' => 'src_street', + 'ca2.street_address' => 'dst_street' + ); + + if($select) { + $join .= " INNER JOIN civicrm_contact cc1 ON cc1.id = pn.entity_id1"; + $join .= " INNER JOIN civicrm_contact cc2 ON cc2.id = pn.entity_id2"; + $join .= " LEFT JOIN civicrm_email ce1 ON (ce1.contact_id = pn.entity_id1 AND ce1.is_primary = 1 )"; + $join .= " LEFT JOIN civicrm_email ce2 ON (ce2.contact_id = pn.entity_id2 AND ce2.is_primary = 1 )"; + $join .= " LEFT JOIN civicrm_address ca1 ON (ca1.contact_id = pn.entity_id1 AND ca1.is_primary = 1 )"; + $join .= " LEFT JOIN civicrm_address ca2 ON (ca2.contact_id = pn.entity_id2 AND ca2.is_primary = 1 )"; + } + $iTotal = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, $join, $whereClause); + foreach ($_REQUEST['order'] as $orderInfo) { + if (!empty($orderInfo['column'])) { + $orderColumnNumber = $orderInfo['column']; + $dir = $orderInfo['dir']; + } + } + $columnDetails = CRM_Utils_Array::value($orderColumnNumber, $_REQUEST['columns']); + if(!empty($columnDetails)) { + switch ($columnDetails['data']) { + case 'src': + $whereClause .= " ORDER BY cc1.display_name {$dir}"; + break; + case 'src_email': + $whereClause .= " ORDER BY ce1.email {$dir}"; + break; + case 'src_street': + $whereClause .= " ORDER BY ca1.street_address {$dir}"; + break; + case 'src_postcode': + $whereClause .= " ORDER BY ca1.postal_code {$dir}"; + break; + case 'dst': + $whereClause .= " ORDER BY cc2.display_name {$dir}"; + break; + case 'dst_email': + $whereClause .= " ORDER BY ce2.email {$dir}"; + break; + case 'dst_street': + $whereClause .= " ORDER BY ca2.street_address {$dir}"; + break; + case 'dst_postcode': + $whereClause .= " ORDER BY ca2.postal_code {$dir}"; + break; + default: + break; + } + } - foreach ($mainContacts as $mainId => $main) { - $searchRows[$mainId]['src'] = CRM_Utils_System::href($main['srcName'], 'civicrm/contact/view', "reset=1&cid={$main['srcID']}"); - $searchRows[$mainId]['dst'] = CRM_Utils_System::href($main['dstName'], 'civicrm/contact/view', "reset=1&cid={$main['dstID']}"); - $searchRows[$mainId]['weight'] = CRM_Utils_Array::value('weight', $main); + $dupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $whereClause, $offset, $rowCount, $select); + $iFilteredTotal = CRM_Core_DAO::singleValueQuery("SELECT FOUND_ROWS()"); + + $count = 0; + foreach ($dupePairs as $key => $pairInfo) { + $pair =& $pairInfo['data']; + $srcContactSubType = CRM_Utils_Array::value('src_contact_sub_type', $pairInfo); + $dstContactSubType = CRM_Utils_Array::value('dst_contact_sub_type', $pairInfo); + $srcTypeImage = CRM_Contact_BAO_Contact_Utils::getImage($srcContactSubType ? + $srcContactSubType : $pairInfo['src_contact_type'], + FALSE, + $pair['srcID'] + ); + $dstTypeImage = CRM_Contact_BAO_Contact_Utils::getImage($dstContactSubType ? + $dstContactSubType : $pairInfo['dst_contact_type'], + FALSE, + $pair['dstID'] + ); - if (!empty($main['canMerge'])) { - $mergeParams = "reset=1&cid={$main['srcID']}&oid={$main['dstID']}&action=update&rgid={$rgid}"; + $searchRows[$count]['is_selected'] = $pairInfo['is_selected']; + $searchRows[$count]['is_selected_input'] = ""; + $searchRows[$count]['src_image'] = $srcTypeImage; + $searchRows[$count]['src'] = CRM_Utils_System::href($pair['srcName'], 'civicrm/contact/view', "reset=1&cid={$pair['srcID']}"); + $searchRows[$count]['src_email'] = CRM_Utils_Array::value('src_email', $pairInfo); + $searchRows[$count]['src_street'] = CRM_Utils_Array::value('src_street', $pairInfo); + $searchRows[$count]['src_postcode'] = CRM_Utils_Array::value('src_postcode', $pairInfo); + $searchRows[$count]['dst_image'] = $dstTypeImage; + $searchRows[$count]['dst'] = CRM_Utils_System::href($pair['dstName'], 'civicrm/contact/view', "reset=1&cid={$pair['dstID']}"); + $searchRows[$count]['dst_email'] = CRM_Utils_Array::value('dst_email', $pairInfo); + $searchRows[$count]['dst_street'] = CRM_Utils_Array::value('dst_street', $pairInfo); + $searchRows[$count]['dst_postcode'] = CRM_Utils_Array::value('dst_postcode', $pairInfo); + $searchRows[$count]['conflicts'] = CRM_Utils_Array::value('conflicts', $pair); + $searchRows[$count]['weight'] = CRM_Utils_Array::value('weight', $pair); + + if (!empty($pair['canMerge'])) { + $mergeParams = "reset=1&cid={$pair['srcID']}&oid={$pair['dstID']}&action=update&rgid={$rgid}"; if ($gid) { $mergeParams .= "&gid={$gid}"; } - $searchRows[$mainId]['actions'] = '' . ts('merge') . ''; - $searchRows[$mainId]['actions'] .= "" . ts('not a duplicate') . ""; + $searchRows[$count]['actions'] = CRM_Utils_System::href(ts('merge'), 'civicrm/contact/merge', $mergeParams); + $searchRows[$count]['actions'] .= " | " . ts('not a duplicate') . ""; } else { - $searchRows[$mainId]['actions'] = '' . ts('Insufficient access rights - cannot merge') . ''; + $searchRows[$count]['actions'] = '' . ts('Insufficient access rights - cannot merge') . ''; } + $count++; } - CRM_Utils_System::setHttpHeader('Content-Type', 'application/json'); - echo CRM_Utils_JSON::encodeDataTableSelector($searchRows, $sEcho, $iTotal, $iFilteredTotal, $selectorElements); + header('Content-Type: application/json'); + echo CRM_Utils_JSON::encodeDataTable($searchRows, $iTotal, $iFilteredTotal, $selectorElements); CRM_Utils_System::civiExit(); } @@ -806,6 +947,41 @@ LIMIT {$offset}, {$rowCount} CRM_Utils_JSON::output($addressVal); } + static function toggleDedupeSelect() { + $rgid = CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer'); + $gid = CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer'); + $pnid = $_REQUEST['pnid']; + $isSelected = CRM_Utils_Type::escape($_REQUEST['is_selected'], 'Boolean'); + + $contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type'); + $cacheKeyString = "merge $contactType"; + $cacheKeyString .= $rgid ? "_{$rgid}" : '_0'; + $cacheKeyString .= $gid ? "_{$gid}" : '_0'; + + $params = array( + 1 => array($isSelected, 'Boolean'), + 3 => array("$cacheKeyString%", 'String') // using % to address rows with conflicts as well + ); + + //check pnid is_array or integer + $whereClause = NULL; + if (is_array($pnid) && !CRM_Utils_Array::crmIsEmptyArray($pnid)) { + $pnid = implode(', ', $pnid); + $pnid = CRM_Utils_Type::escape($pnid, 'String'); + $whereClause = " id IN ( {$pnid} ) "; + } + else { + $pnid = CRM_Utils_Type::escape($pnid, 'Integer'); + $whereClause = " id = %2"; + $params[2] = array($pnid, 'Integer'); + } + + $sql = "UPDATE civicrm_prevnext_cache SET is_selected = %1 WHERE {$whereClause} AND cacheKey LIKE %3"; + CRM_Core_DAO::executeQuery($sql, $params); + + CRM_Utils_System::civiExit(); + } + /** * Retrieve contact relationships. */ diff --git a/CRM/Contact/Page/DedupeFind.php b/CRM/Contact/Page/DedupeFind.php index f401b01405..0129be19d5 100644 --- a/CRM/Contact/Page/DedupeFind.php +++ b/CRM/Contact/Page/DedupeFind.php @@ -94,7 +94,7 @@ class CRM_Contact_Page_DedupeFind extends CRM_Core_Page_Basic { elseif ($action & CRM_Core_Action::MAP) { // do a batch merge if requested $rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0); - $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, TRUE); + $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, 75); $skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0); $skippedCount = $skippedCount + count($result['skipped']); @@ -157,6 +157,9 @@ class CRM_Contact_Page_DedupeFind extends CRM_Core_Page_Basic { if ($rgid) { $sourceParams .= "&rgid={$rgid}"; } + if ($context == 'conflicts') { + $sourceParams .= "&selected=1"; + } $this->assign('sourceUrl', CRM_Utils_System::url('civicrm/ajax/dedupefind', $sourceParams, FALSE, NULL, FALSE)); @@ -165,11 +168,31 @@ class CRM_Contact_Page_DedupeFind extends CRM_Core_Page_Basic { $cacheKeyString .= $rgid ? "_{$rgid}" : '_0'; $cacheKeyString .= $gid ? "_{$gid}" : '_0'; + $stats = CRM_Dedupe_Merger::getMergeStatsMsg($cacheKeyString); + if ($stats) { + CRM_Core_Session::setStatus($stats); + // reset so we not displaying same message again + CRM_Dedupe_Merger::resetMergeStats($cacheKeyString); + } $join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND pn.entity_id2 = de.contact_id2 )"; $where = "de.id IS NULL"; + if ($context == 'conflicts') { + $where .= " AND pn.is_selected = 1"; + } $this->_mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where); if (empty($this->_mainContacts)) { + if ($context == 'conflicts') { + // if the current screen was intended to list only selected contacts, move back to full dupe list + $sourceParams = 'reset=1&action=update'; + if ($gid) { + $sourceParams .= "&gid={$gid}"; + } + if ($rgid) { + $sourceParams .= "&rgid={$rgid}"; + } + CRM_Utils_System::redirect(CRM_Utils_System::url(CRM_Utils_System::currentPath(), $sourceParams)); + } if ($gid) { $foundDupes = $this->get("dedupe_dupes_$gid"); if (!$foundDupes) { diff --git a/CRM/Contact/Page/DedupeMerge.php b/CRM/Contact/Page/DedupeMerge.php new file mode 100644 index 0000000000..a3e393f66a --- /dev/null +++ b/CRM/Contact/Page/DedupeMerge.php @@ -0,0 +1,129 @@ +runAllViaWeb(); + } else { + CRM_Core_Session::setStatus(ts('Nothing to merge.')); + } + + // parent run + return parent::run(); + } + + static function getRunner() { + $rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0); + $gid = CRM_Utils_Request::retrieve('gid', 'Positive', $this, FALSE, 0); + $action = CRM_Utils_Request::retrieve('action', 'String', CRM_Core_DAO::$_nullObject); + $mode = CRM_Utils_Request::retrieve('mode', 'String', CRM_Core_DAO::$_nullObject, FALSE, 'safe'); + + $contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type'); + $cacheKeyString = "merge {$contactType}"; + $cacheKeyString .= $rgid ? "_{$rgid}" : '_0'; + $cacheKeyString .= $gid ? "_{$gid}" : '_0'; + + // Setup the Queue + $queue = CRM_Queue_Service::singleton()->create(array( + 'name' => $cacheKeyString, + 'type' => 'Sql', + 'reset' => TRUE, + )); + + $where = NULL; + if ($action == CRM_Core_Action::MAP) { + $where = "pn.is_selected = 1"; + $isSelected = 1; + } else { + // else merge all (2) + $isSelected = 2; + } + + $urlQry = "reset=1&action=update&rgid={$rgid}"; + $urlQry = $gid ? ($urlQry . "&gid={$gid}") : $urlQry; + + $total = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, NULL, $where); + if ($total <= 0) { + // Nothing to do. + CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry)); + } + + // reset merge stats, so we compute new stats + CRM_Dedupe_Merger::resetMergeStats($cacheKeyString); + + for ($i = 1; $i <= ceil($total/self::BATCHLIMIT); $i++) { + $task = new CRM_Queue_Task( + array ('CRM_Contact_Page_DedupeMerge', 'callBatchMerge'), + array($rgid, $gid, $mode, TRUE, self::BATCHLIMIT, $isSelected), + "Processed " . $i*self::BATCHLIMIT . " pair of duplicates out of " . $total + ); + + // Add the Task to the Queue + $queue->createItem($task); + } + + // Setup the Runner + $urlQry .= "&context=conflicts"; + $runner = new CRM_Queue_Runner(array( + 'title' => ts('Merging Duplicates..'), + 'queue' => $queue, + 'errorMode'=> CRM_Queue_Runner::ERROR_ABORT, + 'onEndUrl' => CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry, TRUE, NULL, FALSE), + )); + + return $runner; + } + + /** + * Collect Mailchimp data into temporary working table. + */ + static function callBatchMerge(CRM_Queue_TaskContext $ctx, $rgid, $gid = NULL, $mode = 'safe', $autoFlip = TRUE, $batchLimit = 1, $isSelected = 2) { + $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, $mode, $autoFlip, $batchLimit, $isSelected); + + return CRM_Queue_Task::TASK_SUCCESS; + } +} + diff --git a/CRM/Core/BAO/PrevNextCache.php b/CRM/Core/BAO/PrevNextCache.php index b9a3e8ea14..90bc832f07 100644 --- a/CRM/Core/BAO/PrevNextCache.php +++ b/CRM/Core/BAO/PrevNextCache.php @@ -153,12 +153,48 @@ WHERE cacheKey = %3 AND if (isset($cacheKey)) { $sql .= " AND cacheKey LIKE %4"; - $params[4] = array("{$cacheKey}%", 'String'); + $params[4] = array("{$cacheKey}%", 'String'); // used % to address any row with conflict-cacheKey e.g "merge Individual_8_0_conflicts" } CRM_Core_DAO::executeQuery($sql, $params); } + static function markConflict($id1, $id2, $cacheKey, $conflicts) { + if (empty($cacheKey) || empty($conflicts)) { + return FALSE; + } + + $sql = "SELECT pn.* + FROM civicrm_prevnext_cache pn + WHERE + ((pn.entity_id1 = %1 AND pn.entity_id2 = %2) OR (pn.entity_id1 = %2 AND pn.entity_id2 = %1)) AND + (cacheKey = %3 OR cacheKey = %4)"; + $params = array( + 1 => array($id1, 'Integer'), + 2 => array($id2, 'Integer'), + 3 => array("{$cacheKey}", 'String'), + 4 => array("{$cacheKey}_conflicts", 'String'), + ); + $pncFind = CRM_Core_DAO::executeQuery($sql, $params); + + while ($pncFind->fetch()) { + $data = $pncFind->data; + if (!empty($data)) { + $data = unserialize($data); + $data['conflicts'] = implode(",", array_keys($conflicts)); + + $pncUp = new CRM_Core_DAO_PrevNextCache(); + $pncUp->id = $pncFind->id; + if ($pncUp->find(TRUE)) { + $pncUp->data = serialize($data); + $pncUp->cacheKey = "{$cacheKey}_conflicts"; + $pncUp->save(); + } + } + } + return TRUE; + } + /** * @param $cacheKey * @param NULL $join @@ -168,14 +204,25 @@ WHERE cacheKey = %3 AND * * @return array */ - public static function retrieve($cacheKey, $join = NULL, $where = NULL, $offset = 0, $rowCount = 0) { + public static function retrieve($cacheKey, $join = NULL, $where = NULL, $offset = 0, $rowCount = 0, $select = array()) { + $selectString = 'pn.*'; + if(!empty($select)) { + $aliasArray = array(); + foreach ($select as $column => $alias) { + $aliasArray[] = $column.' as '.$alias; + } + $selectString .= " , ".implode(' , ', $aliasArray); + } $query = " -SELECT data +SELECT SQL_CALC_FOUND_ROWS {$selectString} FROM civicrm_prevnext_cache pn {$join} -WHERE cacheKey = %1 +WHERE (pn.cacheKey = %1 OR pn.cacheKey = %2) "; - $params = array(1 => array($cacheKey, 'String')); + $params = array( + 1 => array($cacheKey, 'String'), + 2 => array("{$cacheKey}_conflicts", 'String') + ); if ($where) { $query .= " AND {$where}"; @@ -190,19 +237,36 @@ WHERE cacheKey = %1 $dao = CRM_Core_DAO::executeQuery($query, $params); - $main = array(); + $main = array(); + $count = 0; while ($dao->fetch()) { if (self::is_serialized($dao->data)) { - $main[] = unserialize($dao->data); + $main[$count] = unserialize($dao->data); } else { - $main[] = $dao->data; + $main[$count] = $dao->data; + } + + if (!empty($select)) { + $extraData = array(); + foreach ($select as $dfield => $sfield) { + $extraData[$sfield] = $dao->$sfield; + } + $main[$count] = array( + 'prevnext_id' => $dao->id, + 'is_selected' => $dao->is_selected, + 'entity_id1' => $dao->entity_id1, + 'entity_id2' => $dao->entity_id2, + 'data' => $main[$count], + ); + $main[$count] = array_merge($main[$count], $extraData); } + $count++; } return $main; } - + /** * @param $string * @@ -235,13 +299,16 @@ WHERE cacheKey = %1 $query = " SELECT COUNT(*) FROM civicrm_prevnext_cache pn {$join} -WHERE cacheKey $op %1 +WHERE (pn.cacheKey $op %1 OR pn.cacheKey $op %2) "; if ($where) { $query .= " AND {$where}"; } - $params = array(1 => array($cacheKey, 'String')); + $params = array( + 1 => array($cacheKey, 'String'), + 2 => array("{$cacheKey}_conflicts", 'String') + ); return (int) CRM_Core_DAO::singleValueQuery($query, $params, TRUE, FALSE); } diff --git a/CRM/Core/xml/Menu/Contact.xml b/CRM/Core/xml/Menu/Contact.xml index 729452ed1a..b1ddb7d330 100644 --- a/CRM/Core/xml/Menu/Contact.xml +++ b/CRM/Core/xml/Menu/Contact.xml @@ -355,6 +355,12 @@ CRM_Contact_Page_AJAX::getDedupes merge duplicate contacts + + civicrm/contact/dedupemerge + Batch Merge Duplicate Contacts + CRM_Contact_Page_DedupeMerge + merge duplicate contacts + civicrm/dedupe/exception Dedupe Exceptions @@ -378,6 +384,11 @@ CRM_Contact_Page_AJAX::selectUnselectContacts access CiviCRM + + civicrm/ajax/toggleDedupeSelect + CRM_Contact_Page_AJAX::toggleDedupeSelect + merge duplicate contacts + civicrm/activity/sms/add action=add diff --git a/CRM/Dedupe/Merger.php b/CRM/Dedupe/Merger.php index 119164cbbb..1d25f08410 100644 --- a/CRM/Dedupe/Merger.php +++ b/CRM/Dedupe/Merger.php @@ -569,7 +569,7 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m * * @return array|bool */ - public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $autoFlip = TRUE, $redirectForPerformance = FALSE) { + public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $autoFlip = TRUE, $batchLimit = 1, $isSelected = 2) { $contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type'); $cacheKeyString = "merge {$contactType}"; $cacheKeyString .= $rgid ? "_{$rgid}" : '_0'; @@ -577,11 +577,16 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m $join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND pn.entity_id2 = de.contact_id2 )"; - $limit = $redirectForPerformance ? 75 : 1; - $where = "de.id IS NULL LIMIT {$limit}"; + $where = "de.id IS NULL"; + if ($isSelected === 0 || $isSelected === 1) { + $where .= " AND pn.is_selected = {$isSelected}"; + }// else consider all dupe pairs + $where .= " LIMIT {$batchLimit}"; + + $redirectForPerformance = ($batchLimit > 1) ? TRUE : FALSE; $dupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where); - if (empty($dupePairs) && !$redirectForPerformance) { + if (empty($dupePairs) && !$redirectForPerformance && $isSelected == 2) { // If we haven't found any dupes, probably cache is empty. // Try filling cache and give another try. CRM_Core_BAO_PrevNextCache::refillCache($rgid, $gid, $cacheKeyString); @@ -596,6 +601,65 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m return CRM_Dedupe_Merger::merge($dupePairs, $cacheParams, $mode, $autoFlip, $redirectForPerformance); } + static function updateMergeStats($cacheKeyString, $result = array()) { + // gather latest stats + $merged = count($result['merged']); + $skipped = count($result['skipped']); + + if ($merged <= 0 && $skipped <= 0) { + return; + } + + // get previous stats + $previousStats = CRM_Core_BAO_PrevNextCache::retrieve("{$cacheKeyString}_stats"); + if (!empty($previousStats)) { + if ($previousStats[0]['merged']) { + $merged = $merged + $previousStats[0]['merged']; + } + if ($previousStats[0]['skipped']) { + $skipped = $skipped + $previousStats[0]['skipped']; + } + } + + // delete old stats + CRM_Dedupe_Merger::resetMergeStats($cacheKeyString); + + // store the updated stats + $data = array( + 'merged' => $merged, + 'skipped' => $skipped, + ); + $data = CRM_Core_DAO::escapeString(serialize($data)); + + $values = array(); + $values[] = " ( 'civicrm_contact', 0, 0, '{$cacheKeyString}_stats', '$data' ) "; + CRM_Core_BAO_PrevNextCache::setItem($values); + } + + static function resetMergeStats($cacheKeyString) { + return CRM_Core_BAO_PrevNextCache::deleteItem(NULL, "{$cacheKeyString}_stats"); + } + + static function getMergeStats($cacheKeyString) { + $stats = CRM_Core_BAO_PrevNextCache::retrieve("{$cacheKeyString}_stats"); + if (!empty($stats)) { + $stats = $stats[0]; + } + return $stats; + } + + static function getMergeStatsMsg($cacheKeyString) { + $msg = ''; + $stats = CRM_Dedupe_Merger::getMergeStats($cacheKeyString); + if (!empty($stats['merged'])) { + $msg = "{$stats['merged']} " . ts(' Contact(s) were merged. '); + } + if (!empty($stats['skipped'])) { + $msg .= $stats['skipped'] . ts(' Contact(s) were skipped.'); + } + return $msg; + } + /** * Merge given set of contacts. Performs core operation. * @@ -656,7 +720,8 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m $migrationInfo['rows'] = &$rowsElementsAndInfo['rows']; // go ahead with merge if there is no conflict - if (!CRM_Dedupe_Merger::skipMerge($mainId, $otherId, $migrationInfo, $mode)) { + $conflicts = array(); + if (!CRM_Dedupe_Merger::skipMerge($mainId, $otherId, $migrationInfo, $mode, $conflicts)) { CRM_Dedupe_Merger::moveAllBelongings($mainId, $otherId, $migrationInfo); $resultStats['merged'][] = array('main_id' => $mainId, 'other_id' => $otherId); } @@ -664,10 +729,14 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m $resultStats['skipped'][] = array('main_id' => $mainId, 'other_id' => $otherId); } - // delete entry from PrevNextCache table so we don't consider the pair next time - // pair may have been flipped, so make sure we delete using both orders - CRM_Core_BAO_PrevNextCache::deletePair($mainId, $otherId, $cacheKeyString); - CRM_Core_BAO_PrevNextCache::deletePair($otherId, $mainId, $cacheKeyString); + // store any conflicts + if (!empty($conflicts)) { + CRM_Core_BAO_PrevNextCache::markConflict($mainId, $otherId, $cacheKeyString, $conflicts); + } else { + // delete entry from PrevNextCache table so we don't consider the pair next time + // pair may have been flipped, so make sure we delete using both orders + CRM_Core_BAO_PrevNextCache::deletePair($mainId, $otherId, $cacheKeyString, TRUE); + } CRM_Core_DAO::freeResult(); unset($rowsElementsAndInfo, $migrationInfo); @@ -685,6 +754,8 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m unset($dupePairs); } } + + CRM_Dedupe_Merger::updateMergeStats($cacheKeyString, $resultStats); return $resultStats; } @@ -705,8 +776,8 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m * * @return bool */ - public static function skipMerge($mainId, $otherId, &$migrationInfo, $mode = 'safe') { - $conflicts = array(); + public static function skipMerge($mainId, $otherId, &$migrationInfo, $mode = 'safe', &$conflicts = array()) { + //$conflicts = array(); $migrationData = array( 'old_migration_info' => $migrationInfo, 'mode' => $mode, @@ -787,11 +858,13 @@ INNER JOIN civicrm_membership membership2 ON membership1.membership_type_id = m $migrationData['fields_in_conflict'] = $conflicts; CRM_Utils_Hook::merge('batch', $migrationData, $mainId, $otherId); $conflicts = $migrationData['fields_in_conflict']; + // allow hook to override / manipulate migrationInfo as well + $migrationInfo = $migrationData['old_migration_info']; if (!empty($conflicts)) { foreach ($conflicts as $key => $val) { if ($val === NULL and $mode == 'safe') { - // un-resolved conflicts still present. Lets skip this merge. + // un-resolved conflicts still present. Lets skip this merge after saving the conflict / reason. return TRUE; } else { diff --git a/CRM/Utils/JSON.php b/CRM/Utils/JSON.php index 01e0b60bbf..169ac62924 100644 --- a/CRM/Utils/JSON.php +++ b/CRM/Utils/JSON.php @@ -94,4 +94,37 @@ class CRM_Utils_JSON { return $sOutput; } + /** + * This function is used to encode data for new dataTable plugin v1.10 and greater + * @return string + * + */ + static function encodeDataTable($params, $iTotal, $iFilteredTotal, $selectorElements) { + $sOutput = '{'; + $sOutput .= '"recordsTotal": ' . $iTotal . ', '; + $sOutput .= '"recordsFiltered": ' . $iFilteredTotal . ', '; + $sOutput .= '"data": [ '; + foreach ($params as $key => $value) { + $addcomma = FALSE; + $sOutput .= "{"; + foreach ($selectorElements as $element) { + if ($addcomma) { + $sOutput .= ","; + } + //CRM-7130 --lets addslashes to only double quotes, + //since we are using it to quote the field value. + //str_replace helps to provide a break for new-line + $sOutput .= '"' . $element . '":' . '"' . addcslashes(str_replace(array("\r\n", "\n", "\r"), '
', $value[$element]), '"\\') . '"'; + + //remove extra spaces and tab character that breaks dataTable CRM-12551 + $sOutput = preg_replace("/\s+/", " ", $sOutput); + $addcomma = TRUE; + } + $sOutput .= "},"; + } + $sOutput = substr_replace($sOutput, "", -1); + $sOutput .= '] }'; + + return $sOutput; + } } diff --git a/templates/CRM/Contact/Page/DedupeFind.tpl b/templates/CRM/Contact/Page/DedupeFind.tpl index bd07359313..76f380bdbb 100644 --- a/templates/CRM/Contact/Page/DedupeFind.tpl +++ b/templates/CRM/Contact/Page/DedupeFind.tpl @@ -25,12 +25,88 @@ *} {if $action eq 2 || $action eq 16}
- +
+
+ {ts}Filter Contacts{/ts} +
+
+
+ + + + + + + + + + + + +
+
+ + 		+
+ + 		+
+ + 		+
+ +
+
+ + 		+
+ + 		+
+ + 		+
+ +
+
+ +
+ Show / Hide columns: + + + + + + + + +

+ + + + + - + + + + + + + + + + + + + + + + + +
{ts}Contact{/ts} 1{ts}Contact{/ts} 2 ({ts}Duplicate{/ts}){ts}Threshold{/ts} 
 {ts}Contact{/ts} 1 {ts}Contact{/ts} 2 ({ts}Duplicate{/ts}){ts}Email{/ts} 1{ts}Email{/ts} 2 ({ts}Duplicate{/ts}){ts}Street Address{/ts} 1{ts}Street Address{/ts} 2 ({ts}Duplicate{/ts}){ts}Postcode{/ts} 1{ts}Postcode{/ts} 2 ({ts}Duplicate{/ts}){ts}Conflicts{/ts}{ts}Threshold{/ts} 
- {include file="CRM/common/jsortable.tpl" sourceUrl=$sourceUrl useAjax=1 } {if $cid} @@ -51,6 +127,27 @@ {if $context eq 'search'} {crmButton href=$backURL icon="close"}{ts}Done{/ts}{/crmButton} +{elseif $context eq 'conflicts'} + {if $gid} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&gid=`$gid`&action=map&mode=aggressive" a=1}{/capture} + {else} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&action=map&mode=aggressive" a=1}{/capture} + {/if} + {ts}Force Merge Selected Duplicates{/ts} + + {if $gid} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&gid=`$gid`&mode=aggressive" a=1}{/capture} + {else} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&mode=aggressive" a=1}{/capture} + {/if} + {ts}Force Merge All Duplicates{/ts} + + {if $gid} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupefind" q="reset=1&action=update&rgid=`$rgid`&gid=`$gid`" a=1}{/capture} + {else} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupefind" q="reset=1&action=update&rgid=`$rgid`" a=1}{/capture} + {/if} + {ts}List All Duplicates{/ts} {else} {if $gid} {capture assign=backURL}{crmURL p="civicrm/contact/dedupefind" q="reset=1&rgid=`$rgid`&gid=`$gid`&action=renew" a=1}{/capture} @@ -62,13 +159,18 @@ {if $gid} - {capture assign=backURL}{crmURL p="civicrm/contact/dedupefind" q="reset=1&rgid=`$rgid`&gid=`$gid`&action=map" a=1}{/capture} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&gid=`$gid`&action=map" a=1}{/capture} {else} - {capture assign=backURL}{crmURL p="civicrm/contact/dedupefind" q="reset=1&rgid=`$rgid`&action=map" a=1}{/capture} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&action=map" a=1}{/capture} {/if} - -
{ts}Batch Merge Duplicates{/ts} - +
{ts}Batch Merge Selected Duplicates{/ts} + + {if $gid} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`&gid=`$gid`" a=1}{/capture} + {else} + {capture assign=backURL}{crmURL p="civicrm/contact/dedupemerge" q="reset=1&rgid=`$rgid`" a=1}{/capture} + {/if} +
{ts}Batch Merge All Duplicates{/ts} {capture assign=backURL}{crmURL p="civicrm/contact/deduperules" q="reset=1" a=1}{/capture} @@ -84,6 +186,148 @@ {include file='CRM/common/dedupe.tpl'} {literal} {/literal} diff --git a/templates/CRM/Contact/Page/DedupeMerge.tpl b/templates/CRM/Contact/Page/DedupeMerge.tpl new file mode 100644 index 0000000000..a270e4f2d0 --- /dev/null +++ b/templates/CRM/Contact/Page/DedupeMerge.tpl @@ -0,0 +1,25 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.5 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2014 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} -- 2.25.1
{ts 1=$main_contacts[$cid]}Merge %1 with{/ts}