X-Git-Url: https://vcs.fsf.org/?a=blobdiff_plain;f=mediagoblin%2Fstorage.py;h=7ada95e1165c3d177a37993066f715bdb4449ff9;hb=7dc3a66f924c734cbee68677e84f5b656eefe280;hp=d994268b94b729231bf0b24ca389a1cb7fe2e3c6;hpb=93b2796c7e89e269d60db8e48bf84a0f2ca88d12;p=mediagoblin.git diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index d994268b..7ada95e1 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -15,10 +15,10 @@ # along with this program. If not, see . import os -import re import shutil import urlparse import uuid +import cloudfiles from werkzeug.utils import secure_filename @@ -28,11 +28,21 @@ from mediagoblin import util # Errors ######## -class Error(Exception): pass -class InvalidFilepath(Error): pass -class NoWebServing(Error): pass -class NotImplementedError(Error): pass +class Error(Exception): + pass + + +class InvalidFilepath(Error): + pass + + +class NoWebServing(Error): + pass + + +class NotImplementedError(Error): + pass ############################################### @@ -117,7 +127,7 @@ class StorageInterface(object): Eg, if the filename doesn't exist: >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) [u'dir1', u'dir2', u'fname.jpg'] - + But if a file does exist, let's get one back with at uuid tacked on: >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg'] @@ -184,7 +194,7 @@ class BasicFileStorage(StorageInterface): """ return os.path.join( self.base_dir, *clean_listy_filepath(filepath)) - + def file_exists(self, filepath): return os.path.exists(self._resolve_filepath(filepath)) @@ -216,7 +226,98 @@ class BasicFileStorage(StorageInterface): return self._resolve_filepath(filepath) +class CloudFilesStorage(StorageInterface): + def __init__(self, **kwargs): + self.param_container = kwargs.get('cloudfiles_container') + self.param_user = kwargs.get('cloudfiles_user') + self.param_api_key = kwargs.get('cloudfiles_api_key') + self.param_host = kwargs.get('cloudfiles_host') + self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet') + + if not self.param_host: + print('No CloudFiles host URL specified, ' + 'defaulting to Rackspace US') + + self.connection = cloudfiles.get_connection( + username=self.param_user, + api_key=self.param_api_key, + servicenet=True if self.param_use_servicenet == 'true' or \ + self.param_use_servicenet == True else False) + + if not self.param_container == \ + self.connection.get_container(self.param_container): + self.container = self.connection.create_container( + self.param_container) + self.container.make_public( + ttl=60 * 60 * 2) + else: + self.container = self.connection.get_container( + self.param_container) + + self.container_uri = self.container.public_uri() + + def _resolve_filepath(self, filepath): + return '/'.join( + clean_listy_filepath(filepath)) + + def file_exists(self, filepath): + try: + object = self.container.get_object( + self._resolve_filepath(filepath)) + return True + except cloudfiles.errors.NoSuchObject: + return False + + def get_file(self, filepath, mode='r'): + try: + obj = self.container.get_object( + self._resolve_filepath(filepath)) + except cloudfiles.errors.NoSuchObject: + obj = self.container.create_object( + self._resolve_filepath(filepath)) + + return obj + + def delete_file(self, filepath): + # TODO: Also delete unused directories if empty (safely, with + # checks to avoid race conditions). + self.container.delete_object(filepath) + + def file_url(self, filepath): + return '/'.join([ + self.container_uri, + self._resolve_filepath(filepath)]) + + class MountStorage(StorageInterface): + """ + Experimental "Mount" virtual Storage Interface + + This isn't an interface to some real storage, instead it's a + redirecting interface, that redirects requests to other + "StorageInterface"s. + + For example, say you have the paths: + + 1. ['user_data', 'cwebber', 'avatar.jpg'] + 2. ['user_data', 'elrond', 'avatar.jpg'] + 3. ['media_entries', '34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg'] + + You could mount media_entries under CloudFileStorage and user_data + under BasicFileStorage. Then 1 would be passed to + BasicFileStorage under the path ['cwebber', 'avatar.jpg'] and 3 + would be passed to CloudFileStorage under + ['34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg']. + + In other words, this is kind of like mounting /home/ and /etc/ + under different filesystems on your operating system... but with + mediagoblin filestorages :) + + To set this up, you currently need to call the mount() method with + the target path and a backend, that shall be available under that + target path. You have to mount things in a sensible order, + especially you can't mount ["a", "b"] before ["a"]. + """ def __init__(self, **kwargs): self.mounttab = {} @@ -225,22 +326,22 @@ class MountStorage(StorageInterface): Mount a new backend under dirpath """ new_ent = clean_listy_filepath(dirpath) - new_ent.append(u'') print "Mounting:", repr(new_ent) - already, rem_1, table, rem_2 = self.resolve_to_backend(new_ent, True) - print "===", repr(already), repr(rem_1), repr(rem_2) + already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True) + print "===", repr(already), repr(rem_1), repr(rem_2), len(table) + + assert (len(rem_2) > 0) or (None not in table), \ + "That path is already mounted" + assert (len(rem_2) > 0) or (len(table)==0), \ + "A longer path is already mounted here" - assert rem_1.pop(-1) == u'', "Internal Error 1" - assert rem_2.pop(-1) == u'', "Internal Error 2" - assert (already is None) or (len(rem_2) > 0), "Already mounted" for part in rem_2: table[part] = {} table = table[part] - assert not table.has_key(None), "Huh? Already mounted?!" table[None] = backend - def resolve_to_backend(self, filepath, extra_info = False): + def _resolve_to_backend(self, filepath, extra_info = False): """ extra_info = True is for internal use! @@ -273,6 +374,12 @@ class MountStorage(StorageInterface): else: return (res_be, res_fp) + def resolve_to_backend(self, filepath): + backend, filepath = self._resolve_to_backend(filepath) + if backend is None: + raise Error("Path not mounted") + return backend, filepath + def __repr__(self, table = None, indent = []): res = [] if table is None: @@ -292,6 +399,34 @@ class MountStorage(StorageInterface): else: return res + def file_exists(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_exists(filepath) + + def get_file(self, filepath, mode='r'): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_file(filepath, mode) + + def delete_file(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.delete_file(filepath) + + def file_url(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_url(filepath) + + def get_local_path(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_local_path(filepath) + + def copy_locally(self, filepath, dest_path): + """ + Need to override copy_locally, because the local_storage + attribute is not correct. + """ + backend, filepath = self.resolve_to_backend(filepath) + backend.copy_locally(filepath, dest_path) + ########### # Utilities @@ -324,43 +459,36 @@ def clean_listy_filepath(listy_filepath): return cleaned_filepath -def storage_system_from_config(paste_config, storage_prefix): +def storage_system_from_config(config_section): """ - Utility for setting up a storage system from the paste app config. + Utility for setting up a storage system from a config section. - Note that a special argument may be passed in to the paste_config - which is "${storage_prefix}_storage_class" which will provide an + Note that a special argument may be passed in to + the config_section which is "storage_class" which will provide an import path to a storage system. This defaults to "mediagoblin.storage:BasicFileStorage" if otherwise undefined. Arguments: - - paste_config: dictionary of config parameters - - storage_prefix: the storage system we're setting up / will be - getting keys/arguments from. For example 'publicstore' will - grab all arguments that are like 'publicstore_FOO'. + - config_section: dictionary of config parameters Returns: An instantiated storage system. Example: storage_system_from_config( - {'publicstore_base_url': '/media/', - 'publicstore_base_dir': '/var/whatever/media/'}, - 'publicstore') + {'base_url': '/media/', + 'base_dir': '/var/whatever/media/'}) Will return: BasicFileStorage( base_url='/media/', base_dir='/var/whatever/media') """ - prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix)) - - config_params = dict( - [(prefix_re.match(key).groups()[0], value) - for key, value in paste_config.iteritems() - if prefix_re.match(key)]) + # This construct is needed, because dict(config) does + # not replace the variables in the config items. + config_params = dict(config_section.iteritems()) - if config_params.has_key('storage_class'): + if 'storage_class' in config_params: storage_class = config_params['storage_class'] config_params.pop('storage_class') else: @@ -368,5 +496,3 @@ def storage_system_from_config(paste_config, storage_prefix): storage_class = util.import_component(storage_class) return storage_class(**config_params) - -