From: eileen <emcnaughton@wikimedia.org>
Date: Tue, 1 Mar 2016 04:48:06 +0000 (+1300)
Subject: CRM-18128 Return more rows per query and use unbuffered query to manage the memory
X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=24c7dffac476df9bcd3b8db304429e98a7529a9a;p=civicrm-core.git

CRM-18128 Return more rows per query and use unbuffered query to manage the memory

The Limit was set in order to protect memory use. However, if means running often slow queries many times for large exports .. trying another way
---

diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php
index 26f4adacb4..7020d53e90 100644
--- a/CRM/Export/BAO/Export.php
+++ b/CRM/Export/BAO/Export.php
@@ -41,7 +41,7 @@ class CRM_Export_BAO_Export {
   // increase this number a lot to avoid making too many queries
   // LIMIT is not much faster than a no LIMIT query
   // CRM-7675
-  const EXPORT_ROW_COUNT = 10000;
+  const EXPORT_ROW_COUNT = 100000;
 
   /**
    * Get Querymode based on ExportMode
@@ -757,17 +757,20 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
 
     // for CRM-3157 purposes
     $i18n = CRM_Core_I18n::singleton();
+
     list($outputColumns, $headerRows, $sqlColumns, $metadata) = self::getExportStructureArrays($returnProperties, $query, $phoneTypes, $imProviders, $contactRelationshipTypes, $relationQuery, $selectedPaymentFields);
 
-    while (1) {
+    $limitReached = FALSE;
+    while (!$limitReached) {
       $limitQuery = "{$queryString} LIMIT {$offset}, {$rowCount}";
-      $dao = CRM_Core_DAO::executeQuery($limitQuery);
-      if ($dao->N <= 0) {
-        break;
-      }
+      $dao = CRM_Core_DAO::executeUnbufferedQuery($limitQuery);
+      // If this is less than our limit by the end of the iteration we do not need to run the query again to
+      // check if some remain.
+      $rowsThisIteration = 0;
 
       while ($dao->fetch()) {
         $count++;
+        $rowsThisIteration++;
         $row = array();
 
         //convert the pseudo constants
@@ -1084,6 +1087,9 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
         }
       }
       $dao->free();
+      if ($rowsThisIteration < self::EXPORT_ROW_COUNT) {
+        $limitReached = TRUE;
+      }
       $offset += $rowCount;
     }