Commit | Line | Data |
---|---|---|
a2468d18 | 1 | # GNU MediaGoblin -- federated, autonomous media hosting |
cf29e8a8 | 2 | # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. |
a2468d18 JW |
3 | # |
4 | # This program is free software: you can redistribute it and/or modify | |
5 | # it under the terms of the GNU Affero General Public License as published by | |
6 | # the Free Software Foundation, either version 3 of the License, or | |
7 | # (at your option) any later version. | |
8 | # | |
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU Affero General Public License for more details. | |
13 | # | |
14 | # You should have received a copy of the GNU Affero General Public License | |
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | ||
a2468d18 | 17 | import shutil |
a2468d18 JW |
18 | import uuid |
19 | ||
20 | from werkzeug.utils import secure_filename | |
21 | ||
152a3bfa | 22 | from mediagoblin.tools import common |
a2468d18 JW |
23 | |
24 | ######## | |
25 | # Errors | |
26 | ######## | |
27 | ||
28 | ||
29 | class Error(Exception): | |
30 | pass | |
31 | ||
32 | ||
33 | class InvalidFilepath(Error): | |
34 | pass | |
35 | ||
36 | ||
37 | class NoWebServing(Error): | |
38 | pass | |
39 | ||
40 | ||
41 | class NotImplementedError(Error): | |
42 | pass | |
43 | ||
44 | ||
45 | ############################################### | |
46 | # Storage interface & basic file implementation | |
47 | ############################################### | |
48 | ||
49 | class StorageInterface(object): | |
50 | """ | |
51 | Interface for the storage API. | |
52 | ||
53 | This interface doesn't actually provide behavior, but it defines | |
54 | what kind of storage patterns subclasses should provide. | |
55 | ||
56 | It is important to note that the storage API idea of a "filepath" | |
57 | is actually like ['dir1', 'dir2', 'file.jpg'], so keep that in | |
58 | mind while reading method documentation. | |
59 | ||
60 | You should set up your __init__ method with whatever keyword | |
61 | arguments are appropriate to your storage system, but you should | |
62 | also passively accept all extraneous keyword arguments like: | |
63 | ||
64 | def __init__(self, **kwargs): | |
65 | pass | |
66 | ||
67 | See BasicFileStorage as a simple implementation of the | |
68 | StorageInterface. | |
69 | """ | |
70 | ||
71 | # Whether this file store is on the local filesystem. | |
72 | local_storage = False | |
73 | ||
74 | def __raise_not_implemented(self): | |
75 | """ | |
76 | Raise a warning about some component not implemented by a | |
77 | subclass of this interface. | |
78 | """ | |
79 | raise NotImplementedError( | |
80 | "This feature not implemented in this storage API implementation.") | |
81 | ||
82 | def file_exists(self, filepath): | |
83 | """ | |
84 | Return a boolean asserting whether or not file at filepath | |
85 | exists in our storage system. | |
86 | ||
87 | Returns: | |
88 | True / False depending on whether file exists or not. | |
89 | """ | |
90 | # Subclasses should override this method. | |
91 | self.__raise_not_implemented() | |
92 | ||
93 | def get_file(self, filepath, mode='r'): | |
94 | """ | |
95 | Return a file-like object for reading/writing from this filepath. | |
96 | ||
97 | Should create directories, buckets, whatever, as necessary. | |
98 | """ | |
99 | # Subclasses should override this method. | |
100 | self.__raise_not_implemented() | |
101 | ||
102 | def delete_file(self, filepath): | |
103 | """ | |
b34d7e1d | 104 | Delete or dereference the file (not directory) at filepath. |
a2468d18 JW |
105 | """ |
106 | # Subclasses should override this method. | |
107 | self.__raise_not_implemented() | |
108 | ||
b34d7e1d SS |
109 | def delete_dir(self, dirpath, recursive=False): |
110 | """Delete the directory at dirpath | |
111 | ||
112 | :param recursive: Usually, a directory must not contain any | |
113 | files for the delete to succeed. If True, containing files | |
114 | and subdirectories within dirpath will be recursively | |
115 | deleted. | |
a2468d18 | 116 | |
b34d7e1d | 117 | :returns: True in case of success, False otherwise. |
a2468d18 JW |
118 | """ |
119 | # Subclasses should override this method. | |
120 | self.__raise_not_implemented() | |
121 | ||
122 | def file_url(self, filepath): | |
123 | """ | |
124 | Get the URL for this file. This assumes our storage has been | |
125 | mounted with some kind of URL which makes this possible. | |
126 | """ | |
127 | # Subclasses should override this method. | |
128 | self.__raise_not_implemented() | |
129 | ||
130 | def get_unique_filepath(self, filepath): | |
131 | """ | |
132 | If a filename at filepath already exists, generate a new name. | |
133 | ||
134 | Eg, if the filename doesn't exist: | |
135 | >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) | |
136 | [u'dir1', u'dir2', u'fname.jpg'] | |
137 | ||
138 | But if a file does exist, let's get one back with at uuid tacked on: | |
139 | >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) | |
140 | [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg'] | |
141 | """ | |
142 | # Make sure we have a clean filepath to start with, since | |
143 | # we'll be possibly tacking on stuff to the filename. | |
144 | filepath = clean_listy_filepath(filepath) | |
145 | ||
146 | if self.file_exists(filepath): | |
147 | return filepath[:-1] + ["%s-%s" % (uuid.uuid4(), filepath[-1])] | |
148 | else: | |
149 | return filepath | |
150 | ||
151 | def get_local_path(self, filepath): | |
152 | """ | |
153 | If this is a local_storage implementation, give us a link to | |
154 | the local filesystem reference to this file. | |
155 | ||
156 | >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg']) | |
157 | u'/path/to/mounting/foo/bar/baz.jpg' | |
158 | """ | |
159 | # Subclasses should override this method, if applicable. | |
160 | self.__raise_not_implemented() | |
161 | ||
162 | def copy_locally(self, filepath, dest_path): | |
163 | """ | |
164 | Copy this file locally. | |
165 | ||
166 | A basic working method for this is provided that should | |
167 | function both for local_storage systems and remote storge | |
168 | systems, but if more efficient systems for copying locally | |
169 | apply to your system, override this method with something more | |
170 | appropriate. | |
171 | """ | |
172 | if self.local_storage: | |
99a54c00 SS |
173 | # Note: this will copy in small chunks |
174 | shutil.copy(self.get_local_path(filepath), dest_path) | |
a2468d18 JW |
175 | else: |
176 | with self.get_file(filepath, 'rb') as source_file: | |
177 | with file(dest_path, 'wb') as dest_file: | |
99a54c00 SS |
178 | # Copy from remote storage in 4M chunks |
179 | shutil.copyfileobj(source_file, dest_file, length=4*1048576) | |
98f6efb0 CAW |
180 | |
181 | def copy_local_to_storage(self, filename, filepath): | |
182 | """ | |
183 | Copy this file from locally to the storage system. | |
2e8fbc8f CAW |
184 | |
185 | This is kind of the opposite of copy_locally. It's likely you | |
186 | could override this method with something more appropriate to | |
187 | your storage system. | |
98f6efb0 CAW |
188 | """ |
189 | with self.get_file(filepath, 'wb') as dest_file: | |
190 | with file(filename, 'rb') as source_file: | |
99a54c00 SS |
191 | # Copy to storage system in 4M chunks |
192 | shutil.copyfileobj(source_file, dest_file, length=4*1048576) | |
a2468d18 JW |
193 | |
194 | ||
195 | ########### | |
196 | # Utilities | |
197 | ########### | |
198 | ||
199 | def clean_listy_filepath(listy_filepath): | |
200 | """ | |
201 | Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and | |
202 | clean out any nastiness from it. | |
203 | ||
204 | ||
205 | >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg']) | |
206 | [u'dir1', u'foo_.._nasty', u'linooks.jpg'] | |
207 | ||
208 | Args: | |
209 | - listy_filepath: a list of filepath components, mediagoblin | |
210 | storage API style. | |
211 | ||
212 | Returns: | |
213 | A cleaned list of unicode objects. | |
214 | """ | |
215 | cleaned_filepath = [ | |
216 | unicode(secure_filename(filepath)) | |
217 | for filepath in listy_filepath] | |
218 | ||
219 | if u'' in cleaned_filepath: | |
220 | raise InvalidFilepath( | |
221 | "A filename component could not be resolved into a usable name.") | |
222 | ||
223 | return cleaned_filepath | |
224 | ||
225 | ||
226 | def storage_system_from_config(config_section): | |
227 | """ | |
228 | Utility for setting up a storage system from a config section. | |
229 | ||
230 | Note that a special argument may be passed in to | |
231 | the config_section which is "storage_class" which will provide an | |
232 | import path to a storage system. This defaults to | |
233 | "mediagoblin.storage:BasicFileStorage" if otherwise undefined. | |
234 | ||
235 | Arguments: | |
236 | - config_section: dictionary of config parameters | |
237 | ||
238 | Returns: | |
239 | An instantiated storage system. | |
240 | ||
241 | Example: | |
242 | storage_system_from_config( | |
243 | {'base_url': '/media/', | |
244 | 'base_dir': '/var/whatever/media/'}) | |
245 | ||
246 | Will return: | |
247 | BasicFileStorage( | |
248 | base_url='/media/', | |
249 | base_dir='/var/whatever/media') | |
250 | """ | |
251 | # This construct is needed, because dict(config) does | |
252 | # not replace the variables in the config items. | |
253 | config_params = dict(config_section.iteritems()) | |
254 | ||
255 | if 'storage_class' in config_params: | |
256 | storage_class = config_params['storage_class'] | |
257 | config_params.pop('storage_class') | |
258 | else: | |
259 | storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage' | |
260 | ||
152a3bfa | 261 | storage_class = common.import_component(storage_class) |
a2468d18 | 262 | return storage_class(**config_params) |
4a791b80 BS |
263 | |
264 | import filestorage |