mediagoblin/storage/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import shutil
  18 import uuid
  19
  20 from werkzeug.utils import secure_filename
  21
  22 from mediagoblin.tools import common
  23
  24 ########
  25 # Errors
  26 ########
  27
  28
  29 class Error(Exception):
  30     pass
  31
  32
  33 class InvalidFilepath(Error):
  34     pass
  35
  36
  37 class NoWebServing(Error):
  38     pass
  39
  40
  41 class NotImplementedError(Error):
  42     pass
  43
  44
  45 ###############################################
  46 # Storage interface & basic file implementation
  47 ###############################################
  48
  49 class StorageInterface(object):
  50     """
  51     Interface for the storage API.
  52
  53     This interface doesn't actually provide behavior, but it defines
  54     what kind of storage patterns subclasses should provide.
  55
  56     It is important to note that the storage API idea of a "filepath"
  57     is actually like ['dir1', 'dir2', 'file.jpg'], so keep that in
  58     mind while reading method documentation.
  59
  60     You should set up your __init__ method with whatever keyword
  61     arguments are appropriate to your storage system, but you should
  62     also passively accept all extraneous keyword arguments like:
  63
  64       def __init__(self, **kwargs):
  65           pass
  66
  67     See BasicFileStorage as a simple implementation of the
  68     StorageInterface.
  69     """
  70
  71     # Whether this file store is on the local filesystem.
  72     local_storage = False
  73
  74     def __raise_not_implemented(self):
  75         """
  76         Raise a warning about some component not implemented by a
  77         subclass of this interface.
  78         """
  79         raise NotImplementedError(
  80             "This feature not implemented in this storage API implementation.")
  81
  82     def file_exists(self, filepath):
  83         """
  84         Return a boolean asserting whether or not file at filepath
  85         exists in our storage system.
  86
  87         Returns:
  88          True / False depending on whether file exists or not.
  89         """
  90         # Subclasses should override this method.
  91         self.__raise_not_implemented()
  92
  93     def get_file(self, filepath, mode='r'):
  94         """
  95         Return a file-like object for reading/writing from this filepath.
  96
  97         Should create directories, buckets, whatever, as necessary.
  98         """
  99         # Subclasses should override this method.
 100         self.__raise_not_implemented()
 101
 102     def delete_file(self, filepath):
 103         """
 104         Delete or dereference the file (not directory) at filepath.
 105         """
 106         # Subclasses should override this method.
 107         self.__raise_not_implemented()
 108
 109     def delete_dir(self, dirpath, recursive=False):
 110         """Delete the directory at dirpath
 111
 112         :param recursive: Usually, a directory must not contain any
 113             files for the delete to succeed. If True, containing files
 114             and subdirectories within dirpath will be recursively
 115             deleted.
 116
 117         :returns: True in case of success, False otherwise.
 118         """
 119         # Subclasses should override this method.
 120         self.__raise_not_implemented()
 121
 122     def file_url(self, filepath):
 123         """
 124         Get the URL for this file.  This assumes our storage has been
 125         mounted with some kind of URL which makes this possible.
 126         """
 127         # Subclasses should override this method.
 128         self.__raise_not_implemented()
 129
 130     def get_unique_filepath(self, filepath):
 131         """
 132         If a filename at filepath already exists, generate a new name.
 133
 134         Eg, if the filename doesn't exist:
 135         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 136         [u'dir1', u'dir2', u'fname.jpg']
 137
 138         But if a file does exist, let's get one back with at uuid tacked on:
 139         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 140         [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
 141         """
 142         # Make sure we have a clean filepath to start with, since
 143         # we'll be possibly tacking on stuff to the filename.
 144         filepath = clean_listy_filepath(filepath)
 145
 146         if self.file_exists(filepath):
 147             return filepath[:-1] + ["%s-%s" % (uuid.uuid4(), filepath[-1])]
 148         else:
 149             return filepath
 150
 151     def get_local_path(self, filepath):
 152         """
 153         If this is a local_storage implementation, give us a link to
 154         the local filesystem reference to this file.
 155
 156         >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
 157         u'/path/to/mounting/foo/bar/baz.jpg'
 158         """
 159         # Subclasses should override this method, if applicable.
 160         self.__raise_not_implemented()
 161
 162     def copy_locally(self, filepath, dest_path):
 163         """
 164         Copy this file locally.
 165
 166         A basic working method for this is provided that should
 167         function both for local_storage systems and remote storge
 168         systems, but if more efficient systems for copying locally
 169         apply to your system, override this method with something more
 170         appropriate.
 171         """
 172         if self.local_storage:
 173             # Note: this will copy in small chunks
 174             shutil.copy(self.get_local_path(filepath), dest_path)
 175         else:
 176             with self.get_file(filepath, 'rb') as source_file:
 177                 with file(dest_path, 'wb') as dest_file:
 178                     # Copy from remote storage in 4M chunks
 179                     shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 180
 181     def copy_local_to_storage(self, filename, filepath):
 182         """
 183         Copy this file from locally to the storage system.
 184
 185         This is kind of the opposite of copy_locally.  It's likely you
 186         could override this method with something more appropriate to
 187         your storage system.
 188         """
 189         with self.get_file(filepath, 'wb') as dest_file:
 190             with file(filename, 'rb') as source_file:
 191                 # Copy to storage system in 4M chunks
 192                 shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 193
 194
 195 ###########
 196 # Utilities
 197 ###########
 198
 199 def clean_listy_filepath(listy_filepath):
 200     """
 201     Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
 202     clean out any nastiness from it.
 203
 204
 205     >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
 206     [u'dir1', u'foo_.._nasty', u'linooks.jpg']
 207
 208     Args:
 209     - listy_filepath: a list of filepath components, mediagoblin
 210       storage API style.
 211
 212     Returns:
 213       A cleaned list of unicode objects.
 214     """
 215     cleaned_filepath = [
 216         unicode(secure_filename(filepath))
 217         for filepath in listy_filepath]
 218
 219     if u'' in cleaned_filepath:
 220         raise InvalidFilepath(
 221             "A filename component could not be resolved into a usable name.")
 222
 223     return cleaned_filepath
 224
 225
 226 def storage_system_from_config(config_section):
 227     """
 228     Utility for setting up a storage system from a config section.
 229
 230     Note that a special argument may be passed in to
 231     the config_section which is "storage_class" which will provide an
 232     import path to a storage system.  This defaults to
 233     "mediagoblin.storage:BasicFileStorage" if otherwise undefined.
 234
 235     Arguments:
 236      - config_section: dictionary of config parameters
 237
 238     Returns:
 239       An instantiated storage system.
 240
 241     Example:
 242       storage_system_from_config(
 243         {'base_url': '/media/',
 244          'base_dir': '/var/whatever/media/'})
 245
 246        Will return:
 247          BasicFileStorage(
 248            base_url='/media/',
 249            base_dir='/var/whatever/media')
 250     """
 251     # This construct is needed, because dict(config) does
 252     # not replace the variables in the config items.
 253     config_params = dict(config_section.iteritems())
 254
 255     if 'storage_class' in config_params:
 256         storage_class = config_params['storage_class']
 257         config_params.pop('storage_class')
 258     else:
 259         storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage'
 260
 261     storage_class = common.import_component(storage_class)
 262     return storage_class(**config_params)
 263
 264 import filestorage