51b46c07151f8bb65f9346cce2f92c47275d26f1
1 # GNU MediaGoblin -- federated, autonomous media hosting
2 # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 from werkzeug
.utils
import secure_filename
22 from mediagoblin
.tools
import common
29 class Error(Exception):
33 class InvalidFilepath(Error
):
37 class NoWebServing(Error
):
41 class NotImplementedError(Error
):
45 ###############################################
46 # Storage interface & basic file implementation
47 ###############################################
49 class StorageInterface(object):
51 Interface for the storage API.
53 This interface doesn't actually provide behavior, but it defines
54 what kind of storage patterns subclasses should provide.
56 It is important to note that the storage API idea of a "filepath"
57 is actually like ['dir1', 'dir2', 'file.jpg'], so keep that in
58 mind while reading method documentation.
60 You should set up your __init__ method with whatever keyword
61 arguments are appropriate to your storage system, but you should
62 also passively accept all extraneous keyword arguments like:
64 def __init__(self, **kwargs):
67 See BasicFileStorage as a simple implementation of the
71 # Whether this file store is on the local filesystem.
74 def __raise_not_implemented(self
):
76 Raise a warning about some component not implemented by a
77 subclass of this interface.
79 raise NotImplementedError(
80 "This feature not implemented in this storage API implementation.")
82 def file_exists(self
, filepath
):
84 Return a boolean asserting whether or not file at filepath
85 exists in our storage system.
88 True / False depending on whether file exists or not.
90 # Subclasses should override this method.
91 self
.__raise
_not
_implemented
()
93 def get_file(self
, filepath
, mode
='r'):
95 Return a file-like object for reading/writing from this filepath.
97 Should create directories, buckets, whatever, as necessary.
99 # Subclasses should override this method.
100 self
.__raise
_not
_implemented
()
102 def delete_file(self
, filepath
):
104 Delete or dereference the file (not directory) at filepath.
106 # Subclasses should override this method.
107 self
.__raise
_not
_implemented
()
109 def delete_dir(self
, dirpath
, recursive
=False):
110 """Delete the directory at dirpath
112 :param recursive: Usually, a directory must not contain any
113 files for the delete to succeed. If True, containing files
114 and subdirectories within dirpath will be recursively
117 :returns: True in case of success, False otherwise.
119 # Subclasses should override this method.
120 self
.__raise
_not
_implemented
()
122 def file_url(self
, filepath
):
124 Get the URL for this file. This assumes our storage has been
125 mounted with some kind of URL which makes this possible.
127 # Subclasses should override this method.
128 self
.__raise
_not
_implemented
()
130 def get_unique_filepath(self
, filepath
):
132 If a filename at filepath already exists, generate a new name.
134 Eg, if the filename doesn't exist:
135 >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
136 [u'dir1', u'dir2', u'fname.jpg']
138 But if a file does exist, let's get one back with at uuid tacked on:
139 >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
140 [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
142 # Make sure we have a clean filepath to start with, since
143 # we'll be possibly tacking on stuff to the filename.
144 filepath
= clean_listy_filepath(filepath
)
146 if self
.file_exists(filepath
):
147 return filepath
[:-1] + ["%s-%s" % (uuid
.uuid4(), filepath
[-1])]
151 def get_local_path(self
, filepath
):
153 If this is a local_storage implementation, give us a link to
154 the local filesystem reference to this file.
156 >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
157 u'/path/to/mounting/foo/bar/baz.jpg'
159 # Subclasses should override this method, if applicable.
160 self
.__raise
_not
_implemented
()
162 def copy_locally(self
, filepath
, dest_path
):
164 Copy this file locally.
166 A basic working method for this is provided that should
167 function both for local_storage systems and remote storge
168 systems, but if more efficient systems for copying locally
169 apply to your system, override this method with something more
172 if self
.local_storage
:
173 # Note: this will copy in small chunks
174 shutil
.copy(self
.get_local_path(filepath
), dest_path
)
176 with self
.get_file(filepath
, 'rb') as source_file
:
177 with
file(dest_path
, 'wb') as dest_file
:
178 # Copy from remote storage in 4M chunks
179 shutil
.copyfileobj(source_file
, dest_file
, length
=4*1048576)
181 def copy_local_to_storage(self
, filename
, filepath
):
183 Copy this file from locally to the storage system.
185 This is kind of the opposite of copy_locally. It's likely you
186 could override this method with something more appropriate to
189 with self
.get_file(filepath
, 'wb') as dest_file
:
190 with
file(filename
, 'rb') as source_file
:
191 # Copy to storage system in 4M chunks
192 shutil
.copyfileobj(source_file
, dest_file
, length
=4*1048576)
194 def get_file_size(self
, filepath
):
196 Return the size of the file in bytes.
198 # Subclasses should override this method.
199 self
.__raise
_not
_implemented
()
206 def clean_listy_filepath(listy_filepath
):
208 Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
209 clean out any nastiness from it.
212 >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
213 [u'dir1', u'foo_.._nasty', u'linooks.jpg']
216 - listy_filepath: a list of filepath components, mediagoblin
220 A cleaned list of unicode objects.
223 unicode(secure_filename(filepath
))
224 for filepath
in listy_filepath
]
226 if u
'' in cleaned_filepath
:
227 raise InvalidFilepath(
228 "A filename component could not be resolved into a usable name.")
230 return cleaned_filepath
233 def storage_system_from_config(config_section
):
235 Utility for setting up a storage system from a config section.
237 Note that a special argument may be passed in to
238 the config_section which is "storage_class" which will provide an
239 import path to a storage system. This defaults to
240 "mediagoblin.storage:BasicFileStorage" if otherwise undefined.
243 - config_section: dictionary of config parameters
246 An instantiated storage system.
249 storage_system_from_config(
250 {'base_url': '/media/',
251 'base_dir': '/var/whatever/media/'})
256 base_dir='/var/whatever/media')
258 # This construct is needed, because dict(config) does
259 # not replace the variables in the config items.
260 config_params
= dict(config_section
.iteritems())
262 if 'storage_class' in config_params
:
263 storage_class
= config_params
['storage_class']
264 config_params
.pop('storage_class')
266 storage_class
= 'mediagoblin.storage.filestorage:BasicFileStorage'
268 storage_class
= common
.import_component(storage_class
)
269 return storage_class(**config_params
)