mediagoblin/storage.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011 Free Software Foundation, Inc
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import re
  19 import shutil
  20 import urlparse
  21 import uuid
  22 import cloudfiles
  23
  24 from werkzeug.utils import secure_filename
  25
  26 from mediagoblin import util
  27
  28 ########
  29 # Errors
  30 ########
  31
  32
  33 class Error(Exception):
  34     pass
  35
  36
  37 class InvalidFilepath(Error):
  38     pass
  39
  40
  41 class NoWebServing(Error):
  42     pass
  43
  44
  45 class NotImplementedError(Error):
  46     pass
  47
  48
  49 ###############################################
  50 # Storage interface & basic file implementation
  51 ###############################################
  52
  53 class StorageInterface(object):
  54     """
  55     Interface for the storage API.
  56
  57     This interface doesn't actually provide behavior, but it defines
  58     what kind of storage patterns subclasses should provide.
  59
  60     It is important to note that the storage API idea of a "filepath"
  61     is actually like ['dir1', 'dir2', 'file.jpg'], so keep that in
  62     mind while reading method documentation.
  63
  64     You should set up your __init__ method with whatever keyword
  65     arguments are appropriate to your storage system, but you should
  66     also passively accept all extraneous keyword arguments like:
  67
  68       def __init__(self, **kwargs):
  69           pass
  70
  71     See BasicFileStorage as a simple implementation of the
  72     StorageInterface.
  73     """
  74
  75     # Whether this file store is on the local filesystem.
  76     local_storage = False
  77
  78     def __raise_not_implemented(self):
  79         """
  80         Raise a warning about some component not implemented by a
  81         subclass of this interface.
  82         """
  83         raise NotImplementedError(
  84             "This feature not implemented in this storage API implementation.")
  85
  86     def file_exists(self, filepath):
  87         """
  88         Return a boolean asserting whether or not file at filepath
  89         exists in our storage system.
  90
  91         Returns:
  92          True / False depending on whether file exists or not.
  93         """
  94         # Subclasses should override this method.
  95         self.__raise_not_implemented()
  96
  97     def get_file(self, filepath, mode='r'):
  98         """
  99         Return a file-like object for reading/writing from this filepath.
 100
 101         Should create directories, buckets, whatever, as necessary.
 102         """
 103         # Subclasses should override this method.
 104         self.__raise_not_implemented()
 105
 106     def delete_file(self, filepath):
 107         """
 108         Delete or dereference the file at filepath.
 109
 110         This might need to delete directories, buckets, whatever, for
 111         cleanliness.  (Be sure to avoid race conditions on that though)
 112         """
 113         # Subclasses should override this method.
 114         self.__raise_not_implemented()
 115
 116     def file_url(self, filepath):
 117         """
 118         Get the URL for this file.  This assumes our storage has been
 119         mounted with some kind of URL which makes this possible.
 120         """
 121         # Subclasses should override this method.
 122         self.__raise_not_implemented()
 123
 124     def get_unique_filepath(self, filepath):
 125         """
 126         If a filename at filepath already exists, generate a new name.
 127
 128         Eg, if the filename doesn't exist:
 129         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 130         [u'dir1', u'dir2', u'fname.jpg']
 131
 132         But if a file does exist, let's get one back with at uuid tacked on:
 133         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 134         [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
 135         """
 136         # Make sure we have a clean filepath to start with, since
 137         # we'll be possibly tacking on stuff to the filename.
 138         filepath = clean_listy_filepath(filepath)
 139
 140         if self.file_exists(filepath):
 141             return filepath[:-1] + ["%s-%s" % (uuid.uuid4(), filepath[-1])]
 142         else:
 143             return filepath
 144
 145     def get_local_path(self, filepath):
 146         """
 147         If this is a local_storage implementation, give us a link to
 148         the local filesystem reference to this file.
 149
 150         >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
 151         u'/path/to/mounting/foo/bar/baz.jpg'
 152         """
 153         # Subclasses should override this method, if applicable.
 154         self.__raise_not_implemented()
 155
 156     def copy_locally(self, filepath, dest_path):
 157         """
 158         Copy this file locally.
 159
 160         A basic working method for this is provided that should
 161         function both for local_storage systems and remote storge
 162         systems, but if more efficient systems for copying locally
 163         apply to your system, override this method with something more
 164         appropriate.
 165         """
 166         if self.local_storage:
 167             shutil.copy(
 168                 self.get_local_path(filepath), dest_path)
 169         else:
 170             with self.get_file(filepath, 'rb') as source_file:
 171                 with file(dest_path, 'wb') as dest_file:
 172                     dest_file.write(source_file.read())
 173
 174
 175 class BasicFileStorage(StorageInterface):
 176     """
 177     Basic local filesystem implementation of storage API
 178     """
 179
 180     local_storage = True
 181
 182     def __init__(self, base_dir, base_url=None, **kwargs):
 183         """
 184         Keyword arguments:
 185         - base_dir: Base directory things will be served out of.  MUST
 186           be an absolute path.
 187         - base_url: URL files will be served from
 188         """
 189         self.base_dir = base_dir
 190         self.base_url = base_url
 191
 192     def _resolve_filepath(self, filepath):
 193         """
 194         Transform the given filepath into a local filesystem filepath.
 195         """
 196         return os.path.join(
 197             self.base_dir, *clean_listy_filepath(filepath))
 198
 199     def file_exists(self, filepath):
 200         return os.path.exists(self._resolve_filepath(filepath))
 201
 202     def get_file(self, filepath, mode='r'):
 203         # Make directories if necessary
 204         if len(filepath) > 1:
 205             directory = self._resolve_filepath(filepath[:-1])
 206             if not os.path.exists(directory):
 207                 os.makedirs(directory)
 208
 209         # Grab and return the file in the mode specified
 210         return open(self._resolve_filepath(filepath), mode)
 211
 212     def delete_file(self, filepath):
 213         # TODO: Also delete unused directories if empty (safely, with
 214         # checks to avoid race conditions).
 215         os.remove(self._resolve_filepath(filepath))
 216
 217     def file_url(self, filepath):
 218         if not self.base_url:
 219             raise NoWebServing(
 220                 "base_url not set, cannot provide file urls")
 221
 222         return urlparse.urljoin(
 223             self.base_url,
 224             '/'.join(clean_listy_filepath(filepath)))
 225
 226     def get_local_path(self, filepath):
 227         return self._resolve_filepath(filepath)
 228
 229
 230 class CloudFilesStorage(StorageInterface):
 231     def __init__(self, **kwargs):
 232         self.param_container = kwargs.get('cloudfiles_container')
 233         self.param_user = kwargs.get('cloudfiles_user')
 234         self.param_api_key = kwargs.get('cloudfiles_api_key')
 235         self.param_host = kwargs.get('cloudfiles_host')
 236         self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet')
 237
 238         if not self.param_host:
 239             print('No CloudFiles host URL specified, '
 240                   'defaulting to Rackspace US')
 241
 242         self.connection = cloudfiles.get_connection(
 243             username=self.param_user,
 244             api_key=self.param_api_key,
 245             servicenet=True if self.param_use_servicenet == 'true' or \
 246                 self.param_use_servicenet == True else False)
 247
 248         if not self.param_container == \
 249                 self.connection.get_container(self.param_container):
 250             self.container = self.connection.create_container(
 251                 self.param_container)
 252             self.container.make_public(
 253                 ttl=60 * 60 * 2)
 254         else:
 255             self.container = self.connection.get_container(
 256                 self.param_container)
 257
 258     def _resolve_filepath(self, filepath):
 259         return '/'.join(
 260             clean_listy_filepath(filepath))
 261
 262     def file_exists(self, filepath):
 263         try:
 264             object = self.container.get_object(
 265                 self._resolve_filepath(filepath))
 266             return True
 267         except cloudfiles.errors.NoSuchObject:
 268             return False
 269
 270     def get_file(self, filepath, mode='r'):
 271         try:
 272             obj = self.container.get_object(
 273                 self._resolve_filepath(filepath))
 274         except cloudfiles.errors.NoSuchObject:
 275             obj = self.container.create_object(
 276                 self._resolve_filepath(filepath))
 277
 278         return obj
 279
 280     def delete_file(self, filepath):
 281         # TODO: Also delete unused directories if empty (safely, with
 282         # checks to avoid race conditions).
 283         self.container.delete_object(filepath)
 284
 285     def file_url(self, filepath):
 286         return self.get_file(filepath).public_uri()
 287
 288
 289 ###########
 290 # Utilities
 291 ###########
 292
 293 def clean_listy_filepath(listy_filepath):
 294     """
 295     Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
 296     clean out any nastiness from it.
 297
 298
 299     >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
 300     [u'dir1', u'foo_.._nasty', u'linooks.jpg']
 301
 302     Args:
 303     - listy_filepath: a list of filepath components, mediagoblin
 304       storage API style.
 305
 306     Returns:
 307       A cleaned list of unicode objects.
 308     """
 309     cleaned_filepath = [
 310         unicode(secure_filename(filepath))
 311         for filepath in listy_filepath]
 312
 313     if u'' in cleaned_filepath:
 314         raise InvalidFilepath(
 315             "A filename component could not be resolved into a usable name.")
 316
 317     return cleaned_filepath
 318
 319
 320 def storage_system_from_config(paste_config, storage_prefix):
 321     """
 322     Utility for setting up a storage system from the paste app config.
 323
 324     Note that a special argument may be passed in to the paste_config
 325     which is "${storage_prefix}_storage_class" which will provide an
 326     import path to a storage system.  This defaults to
 327     "mediagoblin.storage:BasicFileStorage" if otherwise undefined.
 328
 329     Arguments:
 330      - paste_config: dictionary of config parameters
 331      - storage_prefix: the storage system we're setting up / will be
 332        getting keys/arguments from.  For example 'publicstore' will
 333        grab all arguments that are like 'publicstore_FOO'.
 334
 335     Returns:
 336       An instantiated storage system.
 337
 338     Example:
 339       storage_system_from_config(
 340         {'publicstore_base_url': '/media/',
 341          'publicstore_base_dir': '/var/whatever/media/'},
 342         'publicstore')
 343
 344        Will return:
 345          BasicFileStorage(
 346            base_url='/media/',
 347            base_dir='/var/whatever/media')
 348     """
 349     prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix))
 350
 351     config_params = dict(
 352         [(prefix_re.match(key).groups()[0], value)
 353          for key, value in paste_config.iteritems()
 354          if prefix_re.match(key)])
 355
 356     if 'storage_class' in config_params:
 357         storage_class = config_params['storage_class']
 358         config_params.pop('storage_class')
 359     else:
 360         storage_class = "mediagoblin.storage:BasicFileStorage"
 361
 362     storage_class = util.import_component(storage_class)
 363     return storage_class(**config_params)