# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
-import re
import shutil
import urlparse
import uuid
+import cloudfiles
from werkzeug.utils import secure_filename
# Errors
########
-class Error(Exception): pass
-class InvalidFilepath(Error): pass
-class NoWebServing(Error): pass
-class NotImplementedError(Error): pass
+class Error(Exception):
+ pass
+
+
+class InvalidFilepath(Error):
+ pass
+
+
+class NoWebServing(Error):
+ pass
+
+
+class NotImplementedError(Error):
+ pass
###############################################
Eg, if the filename doesn't exist:
>>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
[u'dir1', u'dir2', u'fname.jpg']
-
+
But if a file does exist, let's get one back with at uuid tacked on:
>>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
[u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
"""
return os.path.join(
self.base_dir, *clean_listy_filepath(filepath))
-
+
def file_exists(self, filepath):
return os.path.exists(self._resolve_filepath(filepath))
return self._resolve_filepath(filepath)
+class CloudFilesStorage(StorageInterface):
+ def __init__(self, **kwargs):
+ self.param_container = kwargs.get('cloudfiles_container')
+ self.param_user = kwargs.get('cloudfiles_user')
+ self.param_api_key = kwargs.get('cloudfiles_api_key')
+ self.param_host = kwargs.get('cloudfiles_host')
+ self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet')
+
+ if not self.param_host:
+ print('No CloudFiles host URL specified, '
+ 'defaulting to Rackspace US')
+
+ self.connection = cloudfiles.get_connection(
+ username=self.param_user,
+ api_key=self.param_api_key,
+ servicenet=True if self.param_use_servicenet == 'true' or \
+ self.param_use_servicenet == True else False)
+
+ if not self.param_container == \
+ self.connection.get_container(self.param_container):
+ self.container = self.connection.create_container(
+ self.param_container)
+ self.container.make_public(
+ ttl=60 * 60 * 2)
+ else:
+ self.container = self.connection.get_container(
+ self.param_container)
+
+ self.container_uri = self.container.public_uri()
+
+ def _resolve_filepath(self, filepath):
+ return '/'.join(
+ clean_listy_filepath(filepath))
+
+ def file_exists(self, filepath):
+ try:
+ object = self.container.get_object(
+ self._resolve_filepath(filepath))
+ return True
+ except cloudfiles.errors.NoSuchObject:
+ return False
+
+ def get_file(self, filepath, mode='r'):
+ try:
+ obj = self.container.get_object(
+ self._resolve_filepath(filepath))
+ except cloudfiles.errors.NoSuchObject:
+ obj = self.container.create_object(
+ self._resolve_filepath(filepath))
+
+ return obj
+
+ def delete_file(self, filepath):
+ # TODO: Also delete unused directories if empty (safely, with
+ # checks to avoid race conditions).
+ self.container.delete_object(filepath)
+
+ def file_url(self, filepath):
+ return '/'.join([
+ self.container_uri,
+ self._resolve_filepath(filepath)])
+
+
+class MountStorage(StorageInterface):
+ """
+ Experimental "Mount" virtual Storage Interface
+
+ This isn't an interface to some real storage, instead it's a
+ redirecting interface, that redirects requests to other
+ "StorageInterface"s.
+
+ For example, say you have the paths:
+
+ 1. ['user_data', 'cwebber', 'avatar.jpg']
+ 2. ['user_data', 'elrond', 'avatar.jpg']
+ 3. ['media_entries', '34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg']
+
+ You could mount media_entries under CloudFileStorage and user_data
+ under BasicFileStorage. Then 1 would be passed to
+ BasicFileStorage under the path ['cwebber', 'avatar.jpg'] and 3
+ would be passed to CloudFileStorage under
+ ['34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg'].
+
+ In other words, this is kind of like mounting /home/ and /etc/
+ under different filesystems on your operating system... but with
+ mediagoblin filestorages :)
+
+ To set this up, you currently need to call the mount() method with
+ the target path and a backend, that shall be available under that
+ target path. You have to mount things in a sensible order,
+ especially you can't mount ["a", "b"] before ["a"].
+ """
+ def __init__(self, **kwargs):
+ self.mounttab = {}
+
+ def mount(self, dirpath, backend):
+ """
+ Mount a new backend under dirpath
+ """
+ new_ent = clean_listy_filepath(dirpath)
+
+ print "Mounting:", repr(new_ent)
+ already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True)
+ print "===", repr(already), repr(rem_1), repr(rem_2), len(table)
+
+ assert (len(rem_2) > 0) or (None not in table), \
+ "That path is already mounted"
+ assert (len(rem_2) > 0) or (len(table)==0), \
+ "A longer path is already mounted here"
+
+ for part in rem_2:
+ table[part] = {}
+ table = table[part]
+ table[None] = backend
+
+ def _resolve_to_backend(self, filepath, extra_info = False):
+ """
+ extra_info = True is for internal use!
+
+ Normally, returns the backend and the filepath inside that backend.
+
+ With extra_info = True it returns the last directory node and the
+ remaining filepath from there in addition.
+ """
+ table = self.mounttab
+ filepath = filepath[:]
+ res_fp = None
+ while True:
+ new_be = table.get(None)
+ if (new_be is not None) or res_fp is None:
+ res_be = new_be
+ res_fp = filepath[:]
+ res_extra = (table, filepath[:])
+ # print "... New res: %r, %r, %r" % (res_be, res_fp, res_extra)
+ if len(filepath) == 0:
+ break
+ query = filepath.pop(0)
+ entry = table.get(query)
+ if entry is not None:
+ table = entry
+ res_extra = (table, filepath[:])
+ else:
+ break
+ if extra_info:
+ return (res_be, res_fp) + res_extra
+ else:
+ return (res_be, res_fp)
+
+ def resolve_to_backend(self, filepath):
+ backend, filepath = self._resolve_to_backend(filepath)
+ if backend is None:
+ raise Error("Path not mounted")
+ return backend, filepath
+
+ def __repr__(self, table = None, indent = []):
+ res = []
+ if table is None:
+ res.append("MountStorage<")
+ table = self.mounttab
+ v = table.get(None)
+ if v:
+ res.append(" " * len(indent) + repr(indent) + ": " + repr(v))
+ for k, v in table.iteritems():
+ if k == None:
+ continue
+ res.append(" " * len(indent) + repr(k) + ":")
+ res += self.__repr__(v, indent + [k])
+ if table is self.mounttab:
+ res.append(">")
+ return "\n".join(res)
+ else:
+ return res
+
+ def file_exists(self, filepath):
+ backend, filepath = self.resolve_to_backend(filepath)
+ return backend.file_exists(filepath)
+
+ def get_file(self, filepath, mode='r'):
+ backend, filepath = self.resolve_to_backend(filepath)
+ return backend.get_file(filepath, mode)
+
+ def delete_file(self, filepath):
+ backend, filepath = self.resolve_to_backend(filepath)
+ return backend.delete_file(filepath)
+
+ def file_url(self, filepath):
+ backend, filepath = self.resolve_to_backend(filepath)
+ return backend.file_url(filepath)
+
+ def get_local_path(self, filepath):
+ backend, filepath = self.resolve_to_backend(filepath)
+ return backend.get_local_path(filepath)
+
+ def copy_locally(self, filepath, dest_path):
+ """
+ Need to override copy_locally, because the local_storage
+ attribute is not correct.
+ """
+ backend, filepath = self.resolve_to_backend(filepath)
+ backend.copy_locally(filepath, dest_path)
+
+
###########
# Utilities
###########
return cleaned_filepath
-def storage_system_from_paste_config(paste_config, storage_prefix):
+def storage_system_from_config(config_section):
"""
- Utility for setting up a storage system from the paste app config.
+ Utility for setting up a storage system from a config section.
- Note that a special argument may be passed in to the paste_config
- which is "${storage_prefix}_storage_class" which will provide an
+ Note that a special argument may be passed in to
+ the config_section which is "storage_class" which will provide an
import path to a storage system. This defaults to
"mediagoblin.storage:BasicFileStorage" if otherwise undefined.
Arguments:
- - paste_config: dictionary of config parameters
- - storage_prefix: the storage system we're setting up / will be
- getting keys/arguments from. For example 'publicstore' will
- grab all arguments that are like 'publicstore_FOO'.
+ - config_section: dictionary of config parameters
Returns:
An instantiated storage system.
Example:
- storage_system_from_paste_config(
- {'publicstore_base_url': '/media/',
- 'publicstore_base_dir': '/var/whatever/media/'},
- 'publicstore')
+ storage_system_from_config(
+ {'base_url': '/media/',
+ 'base_dir': '/var/whatever/media/'})
Will return:
BasicFileStorage(
base_url='/media/',
base_dir='/var/whatever/media')
"""
- prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix))
+ # This construct is needed, because dict(config) does
+ # not replace the variables in the config items.
+ config_params = dict(config_section.iteritems())
- config_params = dict(
- [(prefix_re.match(key).groups()[0], value)
- for key, value in paste_config.iteritems()
- if prefix_re.match(key)])
-
- if config_params.has_key('storage_class'):
+ if 'storage_class' in config_params:
storage_class = config_params['storage_class']
config_params.pop('storage_class')
else:
storage_class = util.import_component(storage_class)
return storage_class(**config_params)
-
-
projects / mediagoblin.git / blobdiff