Use six.iteritems() instead of dict.iteritems().

[mediagoblin.git] / mediagoblin / storage / __init__.py

diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py
index 3df56c2ed00d14ea252acd65eb3351ecde02fac8..121097482ad4f393f57b215986791a55a6e37966 100644 (file)
--- a/mediagoblin/storage/__init__.py
+++ b/mediagoblin/storage/__init__.py
@@ -14,9 +14,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import absolute_import
+
 import shutil
 import uuid
 
+import six
+
 from werkzeug.utils import secure_filename
 
 from mediagoblin.tools import common
@@ -101,10 +105,20 @@ class StorageInterface(object):
 
     def delete_file(self, filepath):
         """
-        Delete or dereference the file at filepath.
+        Delete or dereference the file (not directory) at filepath.
+        """
+        # Subclasses should override this method.
+        self.__raise_not_implemented()
+
+    def delete_dir(self, dirpath, recursive=False):
+        """Delete the directory at dirpath
+
+        :param recursive: Usually, a directory must not contain any
+            files for the delete to succeed. If True, containing files
+            and subdirectories within dirpath will be recursively
+            deleted.
 
-        This might need to delete directories, buckets, whatever, for
-        cleanliness.  (Be sure to avoid race conditions on that though)
+        :returns: True in case of success, False otherwise.
         """
         # Subclasses should override this method.
         self.__raise_not_implemented()
@@ -160,12 +174,13 @@ class StorageInterface(object):
         appropriate.
         """
         if self.local_storage:
-            shutil.copy(
-                self.get_local_path(filepath), dest_path)
+            # Note: this will copy in small chunks
+            shutil.copy(self.get_local_path(filepath), dest_path)
         else:
             with self.get_file(filepath, 'rb') as source_file:
                 with file(dest_path, 'wb') as dest_file:
-                    dest_file.write(source_file.read())
+                    # Copy from remote storage in 4M chunks
+                    shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 
     def copy_local_to_storage(self, filename, filepath):
         """
@@ -177,7 +192,15 @@ class StorageInterface(object):
         """
         with self.get_file(filepath, 'wb') as dest_file:
             with file(filename, 'rb') as source_file:
-                dest_file.write(source_file.read())
+                # Copy to storage system in 4M chunks
+                shutil.copyfileobj(source_file, dest_file, length=4*1048576)
+
+    def get_file_size(self, filepath):
+        """
+        Return the size of the file in bytes.
+        """
+        # Subclasses should override this method.
+        self.__raise_not_implemented()
 
 
 ###########
@@ -238,7 +261,7 @@ def storage_system_from_config(config_section):
     """
     # This construct is needed, because dict(config) does
     # not replace the variables in the config items.
-    config_params = dict(config_section.iteritems())
+    config_params = dict(six.iteritems(config_section))
 
     if 'storage_class' in config_params:
         storage_class = config_params['storage_class']
@@ -248,3 +271,5 @@ def storage_system_from_config(config_section):
 
     storage_class = common.import_component(storage_class)
     return storage_class(**config_params)
+
+from . import filestorage