CRM-18128 Return more rows per query and use unbuffered query to manage the memory

The Limit was set in order to protect memory use. However, if means running often slow queries many times for large exports .. trying another way

diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php
index 26f4adacb4008658d0686e613b9fe9e266cc24aa..7020d53e90aeb20397382ce25bc9ffc5a1b02a4b 100644 (file)
--- a/CRM/Export/BAO/Export.php
+++ b/CRM/Export/BAO/Export.php
@@ -41,7 +41,7 @@ class CRM_Export_BAO_Export {
   // increase this number a lot to avoid making too many queries
   // LIMIT is not much faster than a no LIMIT query
   // CRM-7675
-  const EXPORT_ROW_COUNT = 10000;
+  const EXPORT_ROW_COUNT = 100000;
 
   /**
    * Get Querymode based on ExportMode
@@ -757,17 +757,20 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
 
     // for CRM-3157 purposes
     $i18n = CRM_Core_I18n::singleton();
+
     list($outputColumns, $headerRows, $sqlColumns, $metadata) = self::getExportStructureArrays($returnProperties, $query, $phoneTypes, $imProviders, $contactRelationshipTypes, $relationQuery, $selectedPaymentFields);
 
-    while (1) {
+    $limitReached = FALSE;
+    while (!$limitReached) {
       $limitQuery = "{$queryString} LIMIT {$offset}, {$rowCount}";
-      $dao = CRM_Core_DAO::executeQuery($limitQuery);
-      if ($dao->N <= 0) {
-        break;
-      }
+      $dao = CRM_Core_DAO::executeUnbufferedQuery($limitQuery);
+      // If this is less than our limit by the end of the iteration we do not need to run the query again to
+      // check if some remain.
+      $rowsThisIteration = 0;
 
       while ($dao->fetch()) {
         $count++;
+        $rowsThisIteration++;
         $row = array();
 
         //convert the pseudo constants
@@ -1084,6 +1087,9 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
         }
       }
       $dao->free();
+      if ($rowsThisIteration < self::EXPORT_ROW_COUNT) {
+        $limitReached = TRUE;
+      }
       $offset += $rowCount;
     }