Fix bug 647

Reading a file into memory resulted in depletion of memory. Now files
are read and written lazily, by chunks.

diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py
index 532e5bacab6a096862ba3a11f2513087b9d9bbff..61665ea03cd6c7b85d951ceec1fd901ce0616dd2 100644 (file)
--- a/mediagoblin/storage/cloudfiles.py
+++ b/mediagoblin/storage/cloudfiles.py
@@ -193,27 +193,6 @@ class CloudFilesStorageObjectWrapper():
         return self.storage_object.read(*args, **kwargs)
 
     def write(self, data, *args, **kwargs):
-        """
-        write data to the cloudfiles storage object
-
-        The original motivation for this wrapper is to ensure
-        that buffered writing to a cloudfiles storage object does not overwrite
-        any preexisting data.
-
-        Currently this method does not support any write modes except "append".
-        However if we should need it it would be easy implement.
-        """
-        _log.warn(
-            '{0}.write() has bad performance! Use .send instead for now'\
-            .format(self.__class__.__name__))
-
-        if self.storage_object.size and type(data) == str:
-            _log.debug('{0} is > 0 in size, appending data'.format(
-                self.storage_object.name))
-            data = self.read() + data
-
-        _log.debug('Writing {0}'.format(
-            self.storage_object.name))
         self.storage_object.write(data, *args, **kwargs)
 
     def send(self, *args, **kw):

diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py
index f989539c08a81d4f50d57361054a61e63bc295e2..89f4327640d192737355a4746c53d516702008c1 100644 (file)
--- a/mediagoblin/storage/filestorage.py
+++ b/mediagoblin/storage/filestorage.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import io
 import os
 import shutil
 
@@ -24,6 +25,15 @@ from mediagoblin.storage import (
     clean_listy_filepath,
     NoWebServing)
 
+class FileObjectAwareFile(io.FileIO):
+    def write(self, data):
+        if hasattr(data, 'read'):
+            # We can call data.read(). It means that the data is a file-like
+            # object, which should be saved RAM-friendly way
+            shutil.copyfileobj(data, self)
+        else:
+            super(FileObjectAwareFile, self).write(data)
+
 
 class BasicFileStorage(StorageInterface):
     """
@@ -60,7 +70,7 @@ class BasicFileStorage(StorageInterface):
                 os.makedirs(directory)
 
         # Grab and return the file in the mode specified
-        return open(self._resolve_filepath(filepath), mode)
+        return FileObjectAwareFile(self._resolve_filepath(filepath), mode)
 
     def delete_file(self, filepath):
         """Delete file at filepath

diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py
index 541447e45fe9055edbb0156a9ee178a6bcf7578b..a0e1cf90223a963d4e525c7dab1738cc5fccaf79 100644 (file)
--- a/mediagoblin/submit/lib.py
+++ b/mediagoblin/submit/lib.py
@@ -157,7 +157,7 @@ def submit_media(mg_app, user, submitted_file, filename,
     queue_file = prepare_queue_task(mg_app, entry, filename)
 
     with queue_file:
-        queue_file.write(submitted_file.read())
+        queue_file.write(submitted_file)
 
     # Get file size and round to 2 decimal places
     file_size = mg_app.queue_store.get_file_size(