From 2b4c339de6a5762c59054182034793e3b6002ee4 Mon Sep 17 00:00:00 2001
From: Boris Bobrov <breton@cynicmansion.ru>
Date: Mon, 2 Mar 2015 16:44:13 +0300
Subject: [PATCH] Fix bug 647

Reading a file into memory resulted in depletion of memory. Now files
are read and written lazily, by chunks.
---
 mediagoblin/storage/cloudfiles.py  | 21 ---------------------
 mediagoblin/storage/filestorage.py | 12 +++++++++++-
 mediagoblin/submit/lib.py          |  2 +-
 3 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py
index 532e5bac..61665ea0 100644
--- a/mediagoblin/storage/cloudfiles.py
+++ b/mediagoblin/storage/cloudfiles.py
@@ -193,27 +193,6 @@ class CloudFilesStorageObjectWrapper():
         return self.storage_object.read(*args, **kwargs)
 
     def write(self, data, *args, **kwargs):
-        """
-        write data to the cloudfiles storage object
-
-        The original motivation for this wrapper is to ensure
-        that buffered writing to a cloudfiles storage object does not overwrite
-        any preexisting data.
-
-        Currently this method does not support any write modes except "append".
-        However if we should need it it would be easy implement.
-        """
-        _log.warn(
-            '{0}.write() has bad performance! Use .send instead for now'\
-            .format(self.__class__.__name__))
-
-        if self.storage_object.size and type(data) == str:
-            _log.debug('{0} is > 0 in size, appending data'.format(
-                self.storage_object.name))
-            data = self.read() + data
-
-        _log.debug('Writing {0}'.format(
-            self.storage_object.name))
         self.storage_object.write(data, *args, **kwargs)
 
     def send(self, *args, **kw):
diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py
index f989539c..89f43276 100644
--- a/mediagoblin/storage/filestorage.py
+++ b/mediagoblin/storage/filestorage.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import io
 import os
 import shutil
 
@@ -24,6 +25,15 @@ from mediagoblin.storage import (
     clean_listy_filepath,
     NoWebServing)
 
+class FileObjectAwareFile(io.FileIO):
+    def write(self, data):
+        if hasattr(data, 'read'):
+            # We can call data.read(). It means that the data is a file-like
+            # object, which should be saved RAM-friendly way
+            shutil.copyfileobj(data, self)
+        else:
+            super(FileObjectAwareFile, self).write(data)
+
 
 class BasicFileStorage(StorageInterface):
     """
@@ -60,7 +70,7 @@ class BasicFileStorage(StorageInterface):
                 os.makedirs(directory)
 
         # Grab and return the file in the mode specified
-        return open(self._resolve_filepath(filepath), mode)
+        return FileObjectAwareFile(self._resolve_filepath(filepath), mode)
 
     def delete_file(self, filepath):
         """Delete file at filepath
diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py
index 541447e4..a0e1cf90 100644
--- a/mediagoblin/submit/lib.py
+++ b/mediagoblin/submit/lib.py
@@ -157,7 +157,7 @@ def submit_media(mg_app, user, submitted_file, filename,
     queue_file = prepare_queue_task(mg_app, entry, filename)
 
     with queue_file:
-        queue_file.write(submitted_file.read())
+        queue_file.write(submitted_file)
 
     # Get file size and round to 2 decimal places
     file_size = mg_app.queue_store.get_file_size(
-- 
2.25.1