import cloudfiles
import mimetypes
+import logging
+
+_log = logging.getLogger(__name__)
class CloudFilesStorage(StorageInterface):
mimetypes.add_type("video/webm", "webm")
if not self.param_host:
- print('No CloudFiles host URL specified, '
+ _log.info('No CloudFiles host URL specified, '
'defaulting to Rackspace US')
self.connection = cloudfiles.get_connection(
servicenet=True if self.param_use_servicenet == 'true' or \
self.param_use_servicenet == True else False)
+ _log.debug('Connected to {0} (auth: {1})'.format(
+ self.connection.connection.host,
+ self.connection.auth.host))
+
if not self.param_container == \
self.connection.get_container(self.param_container):
self.container = self.connection.create_container(
self.container = self.connection.get_container(
self.param_container)
+ _log.debug('Container: {0}'.format(
+ self.container.name))
+
self.container_uri = self.container.public_uri()
def _resolve_filepath(self, filepath):
def file_exists(self, filepath):
try:
- self.container.get_object( self._resolve_filepath(filepath))
+ self.container.get_object(self._resolve_filepath(filepath))
return True
except cloudfiles.errors.NoSuchObject:
return False
def get_file(self, filepath, *args, **kwargs):
"""
- - Doesn't care about the "mode" argument
+ - Doesn't care about the "mode" argument.
"""
try:
obj = self.container.get_object(
obj = self.container.create_object(
self._resolve_filepath(filepath))
+ # Detect the mimetype ourselves, since some extensions (webm)
+ # may not be universally accepted as video/webm
mimetype = mimetypes.guess_type(
filepath[-1])
- if mimetype:
+ if mimetype[0]:
+ # Set the mimetype on the CloudFiles object
obj.content_type = mimetype[0]
- # this should finally fix the bug #429
- meta_data = {'mime-type' : mimetype[0]}
- obj.metadata = meta_data
+ obj.metadata = {'mime-type': mimetype[0]}
+ else:
+ obj.content_type = 'application/octet-stream'
+ obj.metadata = {'mime-type': 'application/octet-stream'}
return CloudFilesStorageObjectWrapper(obj, *args, **kwargs)
pass
finally:
pass
-
def file_url(self, filepath):
return '/'.join([
self._resolve_filepath(filepath)])
+ def copy_locally(self, filepath, dest_path):
+ """
+ Copy this file locally.
+
+ A basic working method for this is provided that should
+ function both for local_storage systems and remote storge
+ systems, but if more efficient systems for copying locally
+ apply to your system, override this method with something more
+ appropriate.
+ """
+ # Override this method, using the "stream" iterator for efficient streaming
+ with self.get_file(filepath, 'rb') as source_file:
+ with file(dest_path, 'wb') as dest_file:
+ for data in source_file:
+ dest_file.write(data)
+
+ def copy_local_to_storage(self, filename, filepath):
+ """
+ Copy this file from locally to the storage system.
+
+ This is kind of the opposite of copy_locally. It's likely you
+ could override this method with something more appropriate to
+ your storage system.
+ """
+ # It seems that (our implementation of) cloudfiles.write() takes
+ # all existing data and appends write(data) to it, sending the
+ # full monty over the wire everytime. This would of course
+ # absolutely kill chunked writes with some O(1^n) performance
+ # and bandwidth usage. So, override this method and use the
+ # Cloudfile's "send" interface instead.
+ # TODO: Fixing write() still seems worthwhile though.
+ _log.debug('Sending {0} to cloudfiles...'.format(filepath))
+ with self.get_file(filepath, 'wb') as dest_file:
+ with file(filename, 'rb') as source_file:
+ # Copy to storage system in 4096 byte chunks
+ dest_file.send(source_file)
+
class CloudFilesStorageObjectWrapper():
"""
Wrapper for python-cloudfiles's cloudfiles.storage_object.Object
used to circumvent the mystic `medium.jpg` corruption issue, where
we had both python-cloudfiles and PIL doing buffering on both
- ends and that breaking things.
+ ends and causing breakage.
This wrapper currently meets mediagoblin's needs for a public_store
file-like object.
self.storage_object = storage_object
def read(self, *args, **kwargs):
+ _log.debug('Reading {0}'.format(
+ self.storage_object.name))
return self.storage_object.read(*args, **kwargs)
def write(self, data, *args, **kwargs):
Currently this method does not support any write modes except "append".
However if we should need it it would be easy implement.
"""
+ _log.warn(
+ '{0}.write() has bad performance! Use .send instead for now'\
+ .format(self.__class__.__name__))
+
if self.storage_object.size and type(data) == str:
+ _log.debug('{0} is > 0 in size, appending data'.format(
+ self.storage_object.name))
data = self.read() + data
+ _log.debug('Writing {0}'.format(
+ self.storage_object.name))
self.storage_object.write(data, *args, **kwargs)
+ def send(self, *args, **kw):
+ self.storage_object.send(*args, **kw)
+
def close(self):
+ """
+ Not sure we need anything here.
+ """
pass
def __enter__(self):
see self.__enter__()
"""
self.close()
+
+
+ def __iter__(self, **kwargs):
+ """Make CloudFile an iterator, yielding 8192 bytes by default
+
+ This returns a generator object that can be used to getting the
+ object's content in a memory efficient way.
+
+ Warning: The HTTP response is only complete after this generator
+ has raised a StopIteration. No other methods can be called until
+ this has occurred."""
+ return self.storage_object.stream(**kwargs)
projects / mediagoblin.git / blobdiff