From: jitendrapurohit <jitendra.purohit@webaccessglobal.com>
Date: Tue, 17 May 2016 13:25:33 +0000 (+0530)
Subject: Additional fixes for summary reports
X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=bad98dd514b2a8313d4df2632758e21853de16f2;p=civicrm-core.git

Additional fixes for summary reports
---

diff --git a/CRM/Contact/BAO/GroupContact.php b/CRM/Contact/BAO/GroupContact.php
index 78722e001b..599aff3bf9 100644
--- a/CRM/Contact/BAO/GroupContact.php
+++ b/CRM/Contact/BAO/GroupContact.php
@@ -280,7 +280,7 @@ class CRM_Contact_BAO_GroupContact extends CRM_Contact_DAO_GroupContact {
 
     $select = $from = $where = '';
 
-    $select = 'SELECT DISTINCT civicrm_group.id, civicrm_group.title ';
+    $select = 'SELECT civicrm_group.id, civicrm_group.title ';
     $from = ' FROM civicrm_group ';
     $where = " WHERE civicrm_group.is_active = 1 ";
     if ($contactId) {
@@ -292,9 +292,10 @@ class CRM_Contact_BAO_GroupContact extends CRM_Contact_DAO_GroupContact {
     if ($visibility) {
       $where .= " AND civicrm_group.visibility != 'User and User Admin Only'";
     }
+    $groupBy = " GROUP BY civicrm_group.id, civicrm_group.title";
 
     $orderby = " ORDER BY civicrm_group.name";
-    $sql = $select . $from . $where . $orderby;
+    $sql = $select . $from . $where . $groupBy . $orderby;
 
     $group->query($sql);
 
diff --git a/CRM/Contact/BAO/Query.php b/CRM/Contact/BAO/Query.php
index ca8f0291c9..80115faa55 100644
--- a/CRM/Contact/BAO/Query.php
+++ b/CRM/Contact/BAO/Query.php
@@ -4530,8 +4530,13 @@ civicrm_relationship.is_permission_a_b = 0
    * @return string
    */
   public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
+    $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
+    $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
+    if (version_compare($mysqlVersion, '5.7', '<') && (!empty($sqlMode) && !in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode)))) {
+      return '';
+    }
     $groupBy = (array) $groupBy;
-    $regexToExclude = '/(COALESCE|ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN)\(/i';
+    $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN)\(/i';
     foreach ($selectClauses as $key => $val) {
       $aliasArray = preg_split('/ as /i', $val);
       // if more than 1 alias we need to split by ','.
@@ -4547,7 +4552,7 @@ civicrm_relationship.is_permission_a_b = 0
       else {
         list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
         $dateRegex = '/^(DATE_FORMAT|DATE_ADD)/i';
-        // exclude columns which are alreagy included in groupBy and aggregate functions from select
+        // exclude columns which are already included in groupBy and aggregate functions from select
         if (!in_array($selectColumn, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
           if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
             $selectColAlias[] = $alias;
@@ -4723,7 +4728,7 @@ civicrm_relationship.is_permission_a_b = 0
     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
     $from = " FROM civicrm_prevnext_cache pnc INNER JOIN civicrm_contact contact_a ON contact_a.id = pnc.entity_id1 AND pnc.cacheKey = '$cacheKey' " . substr($from, 31);
     $order = " ORDER BY pnc.id";
-    $groupBy = " GROUP BY contact_a.id";
+    $groupBy = " GROUP BY contact_a.id, pnc.id";
     $groupBy .= self::getGroupByFromSelectColumns($this->_select, 'contact_a.id');
     $limit = " LIMIT $offset, $rowCount";
     $query = "$select $from $where $groupBy $order $limit";
diff --git a/CRM/Contribute/Page/ContributionPage.php b/CRM/Contribute/Page/ContributionPage.php
index bfc2e236fb..b219846ced 100644
--- a/CRM/Contribute/Page/ContributionPage.php
+++ b/CRM/Contribute/Page/ContributionPage.php
@@ -675,7 +675,7 @@ WHERE $whereClause";
 SELECT DISTINCT UPPER(LEFT(title, 1)) as sort_name
 FROM civicrm_contribution_page
 WHERE $whereClause
-ORDER BY LEFT(title, 1)
+ORDER BY UPPER(LEFT(title, 1))
 ";
     $dao = CRM_Core_DAO::executeQuery($query, $whereParams);
 
diff --git a/CRM/Event/BAO/Participant.php b/CRM/Event/BAO/Participant.php
index 255b54aed4..bcf797f97e 100644
--- a/CRM/Event/BAO/Participant.php
+++ b/CRM/Event/BAO/Participant.php
@@ -1926,12 +1926,14 @@ WHERE (li.entity_table = 'civicrm_participant' AND li.entity_id = {$participantI
 ";
       CRM_Core_DAO::executeQuery($updateLineItem);
 
+      $fiColumns = "fi.contact_id, fi.description, fi.amount, fi.currency,
+        fi.financial_account_id, fi.status_id, fi.entity_table, fi.entity_id";
       // gathering necessary info to record negative (deselected) financial_item
       $updateFinancialItem = "
-  SELECT fi.*, SUM(fi.amount) as differenceAmt, price_field_value_id, financial_type_id, tax_amount
+  SELECT {$fiColumns}, SUM(fi.amount) as differenceAmt, price_field_value_id, financial_type_id, tax_amount
     FROM civicrm_financial_item fi LEFT JOIN civicrm_line_item li ON (li.id = fi.entity_id AND fi.entity_table = 'civicrm_line_item')
 WHERE (li.entity_table = 'civicrm_participant' AND li.entity_id = {$participantId})
-GROUP BY li.entity_table, li.entity_id, price_field_value_id
+GROUP BY li.entity_table, li.entity_id, price_field_value_id, fi.id, {$fiColumns}
 ";
       $updateFinancialItemInfoDAO = CRM_Core_DAO::executeQuery($updateFinancialItem);
       $trxn = CRM_Core_BAO_FinancialTrxn::getFinancialTrxnId($contributionId, 'DESC', TRUE);
@@ -1943,9 +1945,7 @@ GROUP BY li.entity_table, li.entity_id, price_field_value_id
       while ($updateFinancialItemInfoDAO->fetch()) {
         $updateFinancialItemInfoValues = (array) $updateFinancialItemInfoDAO;
         $updateFinancialItemInfoValues['transaction_date'] = date('YmdHis');
-        // the below params are not needed
-        unset($updateFinancialItemInfoValues['id']);
-        unset($updateFinancialItemInfoValues['created_date']);
+
         // if not submitted and difference is not 0 make it negative
         if (!in_array($updateFinancialItemInfoValues['price_field_value_id'], $submittedFieldValueIds) && $updateFinancialItemInfoValues['differenceAmt'] != 0) {
           // INSERT negative financial_items
diff --git a/CRM/Event/Page/ManageEvent.php b/CRM/Event/Page/ManageEvent.php
index ed741659ac..a661d3ad6b 100644
--- a/CRM/Event/Page/ManageEvent.php
+++ b/CRM/Event/Page/ManageEvent.php
@@ -554,7 +554,7 @@ SELECT count(id)
    SELECT DISTINCT UPPER(LEFT(title, 1)) as sort_name
      FROM civicrm_event
     WHERE $whereClause
- ORDER BY LEFT(title, 1)
+ ORDER BY UPPER(LEFT(title, 1))
 ";
     $dao = CRM_Core_DAO::executeQuery($query, $whereParams);
 
diff --git a/CRM/Mailing/BAO/Mailing.php b/CRM/Mailing/BAO/Mailing.php
index 012a2338c4..5077879916 100644
--- a/CRM/Mailing/BAO/Mailing.php
+++ b/CRM/Mailing/BAO/Mailing.php
@@ -287,7 +287,7 @@ WHERE  c.group_id = {$groupDAO->id}
 
     $query = "REPLACE INTO       I_$job_id (email_id, contact_id)
 
-                    SELECT DISTINCT     $email.id as email_id,
+                    SELECT      $email.id as email_id,
                                         $contact.id as contact_id
                     FROM                $email
                     INNER JOIN          $contact
@@ -312,6 +312,7 @@ WHERE  c.group_id = {$groupDAO->id}
                         AND             $email.on_hold = 0
                         AND             $mg.mailing_id = {$mailing_id}
                         AND             X_$job_id.contact_id IS null
+                        GROUP BY $email.id, $contact.id
                     $order_by";
 
     if ($mode == 'sms') {
@@ -348,7 +349,7 @@ WHERE  c.group_id = {$groupDAO->id}
     // Query prior mailings.
 
     $query = "REPLACE INTO       I_$job_id (email_id, contact_id)
-                    SELECT DISTINCT     $email.id as email_id,
+                    SELECT      $email.id as email_id,
                                         $contact.id as contact_id
                     FROM                $email
                     INNER JOIN          $contact
@@ -370,6 +371,7 @@ WHERE  c.group_id = {$groupDAO->id}
                         AND             $email.on_hold = 0
                         AND             $mg.mailing_id = {$mailing_id}
                         AND             X_$job_id.contact_id IS null
+                        GROUP BY $email.id, $contact.id
                     $order_by";
 
     if ($mode == 'sms') {
@@ -472,7 +474,7 @@ AND    $mg.mailing_id = {$mailing_id}
     // Get the emails with only location override.
 
     $query = "REPLACE INTO       I_$job_id (email_id, contact_id)
-                    SELECT DISTINCT     $email.id as local_email_id,
+                    SELECT              $email.id as local_email_id,
                                         $contact.id as contact_id
                     FROM                $email
                     INNER JOIN          $contact
@@ -494,6 +496,7 @@ AND    $mg.mailing_id = {$mailing_id}
                         AND             $email.on_hold = 0
                         AND             $mg.mailing_id = {$mailing_id}
                         AND             X_$job_id.contact_id IS null
+                        GROUP BY $email.id, $contact.id
                     $order_by";
     if ($mode == "sms") {
       $query = "REPLACE INTO       I_$job_id (phone_id, contact_id)
@@ -2525,8 +2528,11 @@ LEFT JOIN civicrm_mailing_group g ON g.mailing_id   = m.id
             LEFT JOIN   $job ON ( $job.mailing_id = $mailing.id AND $job.is_test = 0 AND $job.parent_id IS NULL )
             LEFT JOIN   civicrm_contact createdContact ON ( civicrm_mailing.created_id = createdContact.id )
             LEFT JOIN   civicrm_contact scheduledContact ON ( civicrm_mailing.scheduled_id = scheduledContact.id )
-            WHERE       $mailingACL $additionalClause
-            GROUP BY    {$groupFromSelect}";
+            WHERE       $mailingACL $additionalClause";
+
+    if (!empty($groupFromSelect)) {
+      $query .= " GROUP BY {$groupFromSelect}";
+    }
 
     if ($sort) {
       $orderBy = trim($sort->orderBy());
diff --git a/CRM/Mailing/Selector/Browse.php b/CRM/Mailing/Selector/Browse.php
index 850db5458d..9ede064a75 100644
--- a/CRM/Mailing/Selector/Browse.php
+++ b/CRM/Mailing/Selector/Browse.php
@@ -598,7 +598,7 @@ LEFT JOIN civicrm_mailing_job ON (civicrm_mailing_job.mailing_id = civicrm_maili
 LEFT JOIN civicrm_contact createdContact ON ( civicrm_mailing.created_id = createdContact.id )
 LEFT JOIN civicrm_contact scheduledContact ON ( civicrm_mailing.scheduled_id = scheduledContact.id )
 WHERE $whereClause
-ORDER BY LEFT(name, 1)
+ORDER BY UPPER(LEFT(name, 1))
 ";
 
     $dao = CRM_Core_DAO::executeQuery($query, $params);
diff --git a/CRM/Report/Form.php b/CRM/Report/Form.php
index c0c2209537..71d511671c 100644
--- a/CRM/Report/Form.php
+++ b/CRM/Report/Form.php
@@ -2642,6 +2642,35 @@ WHERE cg.extends IN ('" . implode("','", $this->_customGroupExtends) . "') AND
     return $sql;
   }
 
+  /**
+   * append select with ANY_VALUE() keyword.
+   *
+   * @param array $selectArray
+   * @param array $groupBy - Columns already included in GROUP By clause.
+   */
+  public function appendSelect($selectClauses, $groupBy) {
+    $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
+    $sqlMode = explode(',', CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode'));
+
+    // Disable only_full_group_by mode for lower sql versions.
+    if (version_compare($mysqlVersion, '5.7', '<') || (!empty($sqlMode) && !in_array('ONLY_FULL_GROUP_BY', $sqlMode))) {
+      $key = array_search('ONLY_FULL_GROUP_BY', $sqlMode);
+      unset($sqlMode[$key]);
+      CRM_Core_DAO::executeQuery("SET SESSION sql_mode = '" . implode(',', $sqlMode) . "'");
+      return;
+    }
+    $groupBy = array_map('trim', (array) $groupBy);
+    $aggregateFunctions = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN)\(/i';
+    foreach ($selectClauses as $key => &$val) {
+      list($selectColumn, $alias) = array_pad(preg_split('/ as /i', $val), 2, NULL);
+      // append ANY_VALUE() keyword
+      if (!in_array($selectColumn, $groupBy) && preg_match($aggregateFunctions, trim($selectColumn)) !== 1) {
+        $val = str_replace($selectColumn, "ANY_VALUE({$selectColumn})", $val);
+      }
+    }
+    $this->_select = "SELECT " . implode(', ', $selectClauses) . " ";
+  }
+
   /**
    * Build group by clause.
    */
diff --git a/CRM/Report/Form/Contribute/Lybunt.php b/CRM/Report/Form/Contribute/Lybunt.php
index da628bd166..db26b325a9 100644
--- a/CRM/Report/Form/Contribute/Lybunt.php
+++ b/CRM/Report/Form/Contribute/Lybunt.php
@@ -550,7 +550,7 @@ class CRM_Report_Form_Contribute_Lybunt extends CRM_Report_Form {
 
   public function groupBy() {
     $this->_groupBy = "GROUP BY  {$this->_aliases['civicrm_contribution']}.contact_id ";
-    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, "{$this->_aliases['civicrm_contribution']}.contact_id");
+    $this->appendSelect($this->_selectClauses, "{$this->_aliases['civicrm_contribution']}.contact_id");
     $this->assign('chartSupported', TRUE);
   }
 
@@ -687,7 +687,7 @@ class CRM_Report_Form_Contribute_Lybunt extends CRM_Report_Form {
       $this->limit();
     }
 
-    $sql = "{$this->_select} {$this->_from} {$this->_where} $limitFilter {$this->_groupBy} {$this->_having} {$this->_rollup}";
+    $sql = "{$this->_select} {$this->_from} {$this->_where} {$limitFilter} {$this->_groupBy} {$this->_having} {$this->_rollup}";
 
     if (!empty($this->_orderByArray)) {
       $this->_orderBy = str_replace('contact_civireport.', 'civicrm_contact_', "ORDER BY ISNULL(civicrm_contribution_contact_id), " . implode(', ', $this->_orderByArray));
diff --git a/CRM/Report/Form/Contribute/SoftCredit.php b/CRM/Report/Form/Contribute/SoftCredit.php
index a174818b50..70b99eb313 100644
--- a/CRM/Report/Form/Contribute/SoftCredit.php
+++ b/CRM/Report/Form/Contribute/SoftCredit.php
@@ -459,9 +459,9 @@ class CRM_Report_Form_Contribute_SoftCredit extends CRM_Report_Form {
 
   public function groupBy() {
     $this->_rollup = 'WITH ROLLUP';
-    $groupFromSelect = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->selectClause, array("{$this->_aliases['civicrm_contribution_soft']}.contact_id", "constituentname.id"));
+    $this->appendSelect($this->selectClause, array("{$this->_aliases['civicrm_contribution_soft']}.contact_id", "constituentname.id"));
     $this->_groupBy = "
-GROUP BY {$this->_aliases['civicrm_contribution_soft']}.contact_id, constituentname.id {$groupFromSelect} {$this->_rollup}";
+GROUP BY {$this->_aliases['civicrm_contribution_soft']}.contact_id, constituentname.id {$this->_rollup}";
   }
 
   public function where() {
diff --git a/CRM/Report/Form/Contribute/Summary.php b/CRM/Report/Form/Contribute/Summary.php
index 883a1a7feb..e4efe70d84 100644
--- a/CRM/Report/Form/Contribute/Summary.php
+++ b/CRM/Report/Form/Contribute/Summary.php
@@ -509,7 +509,7 @@ class CRM_Report_Form_Contribute_Summary extends CRM_Report_Form {
                 !empty($this->_params['group_bys_freq'][$fieldName])
               ) {
 
-                $append = "YEAR({$field['dbAlias']}),";
+                $append = "YEAR({$field['dbAlias']});;";
                 if (in_array(strtolower($this->_params['group_bys_freq'][$fieldName]),
                   array('year')
                 )) {
@@ -537,15 +537,24 @@ class CRM_Report_Form_Contribute_Summary extends CRM_Report_Form {
       ) {
         $this->_rollup = " WITH ROLLUP";
       }
-      $groupBy = $this->_groupBy;
-      $this->_groupBy = "GROUP BY " . implode(', ', $this->_groupBy);
+      $groupBy = array();
+      foreach ($this->_groupBy as $key => $val) {
+        if (strpos($val, ';;') !== false) {
+          $groupBy = array_merge($groupBy, explode(';;', $val));
+        }
+        else {
+          $groupBy[] = $this->_groupBy[$key];
+        }
+      }
+      $this->_groupBy = "GROUP BY " . implode(', ', $groupBy);
     }
     else {
       $groupBy = "{$this->_aliases['civicrm_contact']}.id";
       $this->_groupBy = "GROUP BY {$this->_aliases['civicrm_contact']}.id";
     }
-    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, $groupBy);
     $this->_groupBy .= $this->_rollup;
+    // append select with ANY_VALUE() keyword
+    $this->appendSelect($this->_selectClauses, $groupBy);
   }
 
   /**
diff --git a/CRM/Report/Form/Contribute/Sybunt.php b/CRM/Report/Form/Contribute/Sybunt.php
index e8538005b0..daf36b1e57 100644
--- a/CRM/Report/Form/Contribute/Sybunt.php
+++ b/CRM/Report/Form/Contribute/Sybunt.php
@@ -398,7 +398,7 @@ class CRM_Report_Form_Contribute_Sybunt extends CRM_Report_Form {
     $this->assign('chartSupported', TRUE);
     $fiscalYearOffset = self::fiscalYearOffset("{$this->_aliases['civicrm_contribution']}.receive_date");
     $this->_groupBy = "GROUP BY {$this->_aliases['civicrm_contribution']}.contact_id, {$fiscalYearOffset}";
-    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, array("{$this->_aliases['civicrm_contribution']}.contact_id", $fiscalYearOffset));
+    $this->appendSelect($this->_selectClauses, array("{$this->_aliases['civicrm_contribution']}.contact_id", $fiscalYearOffset));
     $this->_groupBy .= " {$this->_rollup}";
   }
 
diff --git a/CRM/Report/Form/Event/IncomeCountSummary.php b/CRM/Report/Form/Event/IncomeCountSummary.php
index 7f160db484..f502c2d0a4 100644
--- a/CRM/Report/Form/Event/IncomeCountSummary.php
+++ b/CRM/Report/Form/Event/IncomeCountSummary.php
@@ -305,8 +305,8 @@ class CRM_Report_Form_Event_IncomeCountSummary extends CRM_Report_Form_Event {
   public function groupBy() {
     $this->assign('chartSupported', TRUE);
     $this->_rollup = " WITH ROLLUP";
-    $groupFromSelect = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, "{$this->_aliases['civicrm_event']}.id");
-    $this->_groupBy = " GROUP BY {$this->_aliases['civicrm_event']}.id {$groupFromSelect} {$this->_rollup}";
+    $this->appendSelect($this->_selectClauses, "{$this->_aliases['civicrm_event']}.id");
+    $this->_groupBy = " GROUP BY {$this->_aliases['civicrm_event']}.id {$this->_rollup}";
   }
 
   public function postProcess() {
diff --git a/CRM/Report/Form/Mailing/Summary.php b/CRM/Report/Form/Mailing/Summary.php
index 85f9101093..641f9cdc7f 100644
--- a/CRM/Report/Form/Mailing/Summary.php
+++ b/CRM/Report/Form/Mailing/Summary.php
@@ -459,8 +459,12 @@ class CRM_Report_Form_Mailing_Summary extends CRM_Report_Form {
   }
 
   public function groupBy() {
-    $this->_groupBy = "GROUP BY {$this->_aliases['civicrm_mailing']}.id";
-    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, "{$this->_aliases['civicrm_mailing']}.id");
+    $groupBy = array(
+      "{$this->_aliases['civicrm_mailing']}.id",
+      "{$this->_aliases['civicrm_mailing_job']}.end_date",
+    );
+    $this->_groupBy = "GROUP BY " . implode(', ', $groupBy);
+    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, $groupBy);
   }
 
   public function orderBy() {
diff --git a/CRM/Report/Form/Member/Summary.php b/CRM/Report/Form/Member/Summary.php
index de2cba8947..3108adb456 100644
--- a/CRM/Report/Form/Member/Summary.php
+++ b/CRM/Report/Form/Member/Summary.php
@@ -381,8 +381,8 @@ class CRM_Report_Form_Member_Summary extends CRM_Report_Form {
       }
 
       $this->_rollup = ' WITH ROLLUP';
-      $groupFromSelect = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, $this->_groupBy);
-      $this->_groupBy = 'GROUP BY ' . implode(', ', $this->_groupBy) . $groupFromSelect .
+      $this->appendSelect($this->_selectClauses, $this->_groupBy);
+      $this->_groupBy = 'GROUP BY ' . implode(', ', $this->_groupBy) .
         " {$this->_rollup} ";
     }
     else {
diff --git a/CRM/Report/Form/Pledge/Summary.php b/CRM/Report/Form/Pledge/Summary.php
index 2b002ff0b1..a5b4d022d9 100644
--- a/CRM/Report/Form/Pledge/Summary.php
+++ b/CRM/Report/Form/Pledge/Summary.php
@@ -280,7 +280,7 @@ class CRM_Report_Form_Pledge_Summary extends CRM_Report_Form {
       $groupBy = "{$this->_aliases['civicrm_contact']}.id";
       $this->_groupBy = "GROUP BY {$groupBy}";
     }
-    $this->_groupBy .= CRM_Contact_BAO_Query::getGroupByFromSelectColumns($this->_selectClauses, $groupBy);
+    $this->appendSelect($this->_selectClauses, $groupBy);
     $this->_groupBy .= " {$this->_rollup}";
   }