Commit | Line | Data |
---|---|---|
41f446f4 | 1 | # GNU MediaGoblin -- federated, autonomous media hosting |
cf29e8a8 | 2 | # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. |
41f446f4 CAW |
3 | # |
4 | # This program is free software: you can redistribute it and/or modify | |
5 | # it under the terms of the GNU Affero General Public License as published by | |
6 | # the Free Software Foundation, either version 3 of the License, or | |
7 | # (at your option) any later version. | |
8 | # | |
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU Affero General Public License for more details. | |
13 | # | |
14 | # You should have received a copy of the GNU Affero General Public License | |
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | ||
85ead8ac | 17 | from collections import OrderedDict |
64da09e8 | 18 | import logging |
095fbdaf | 19 | import os |
64da09e8 | 20 | |
4a477e24 | 21 | from mediagoblin import mg_globals as mgg |
77ea4c9b CAW |
22 | from mediagoblin.db.util import atomic_update |
23 | from mediagoblin.db.models import MediaEntry | |
24 | from mediagoblin.tools.pluginapi import hook_handle | |
6506b1e2 | 25 | from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ |
8e5f9746 | 26 | |
64da09e8 | 27 | _log = logging.getLogger(__name__) |
41f446f4 | 28 | |
41f446f4 | 29 | |
64712915 JW |
30 | class ProgressCallback(object): |
31 | def __init__(self, entry): | |
32 | self.entry = entry | |
33 | ||
34 | def __call__(self, progress): | |
35 | if progress: | |
36 | self.entry.transcoding_progress = progress | |
37 | self.entry.save() | |
38 | ||
39 | ||
180bdbde | 40 | def create_pub_filepath(entry, filename): |
48a7ba1e | 41 | return mgg.public_store.get_unique_filepath( |
180bdbde | 42 | ['media_entries', |
5c2b8486 | 43 | unicode(entry.id), |
180bdbde E |
44 | filename]) |
45 | ||
64712915 | 46 | |
28f364bd | 47 | class FilenameBuilder(object): |
4774cfa3 BS |
48 | """Easily slice and dice filenames. |
49 | ||
28f364bd | 50 | Initialize this class with an original file path, then use the fill() |
4774cfa3 BS |
51 | method to create new filenames based on the original. |
52 | ||
53 | """ | |
54 | MAX_FILENAME_LENGTH = 255 # VFAT's maximum filename length | |
095fbdaf BS |
55 | |
56 | def __init__(self, path): | |
28f364bd | 57 | """Initialize a builder from an original file path.""" |
095fbdaf BS |
58 | self.dirpath, self.basename = os.path.split(path) |
59 | self.basename, self.ext = os.path.splitext(self.basename) | |
60 | self.ext = self.ext.lower() | |
61 | ||
28f364bd BS |
62 | def fill(self, fmtstr): |
63 | """Build a new filename based on the original. | |
4774cfa3 | 64 | |
28f364bd BS |
65 | The fmtstr argument can include the following: |
66 | {basename} -- the original basename, with the extension removed | |
67 | {ext} -- the original extension, always lowercase | |
68 | ||
69 | If necessary, {basename} will be truncated so the filename does not | |
70 | exceed this class' MAX_FILENAME_LENGTH in length. | |
4774cfa3 BS |
71 | |
72 | """ | |
095fbdaf BS |
73 | basename_len = (self.MAX_FILENAME_LENGTH - |
74 | len(fmtstr.format(basename='', ext=self.ext))) | |
75 | return fmtstr.format(basename=self.basename[:basename_len], | |
76 | ext=self.ext) | |
77 | ||
78 | ||
e6bd03d4 | 79 | |
14565fb7 | 80 | class MediaProcessor(object): |
274a0f67 CAW |
81 | """A particular processor for this media type. |
82 | ||
83 | While the ProcessingManager handles all types of MediaProcessing | |
84 | possible for a particular media type, a MediaProcessor can be | |
85 | thought of as a *particular* processing action for a media type. | |
86 | For example, you may have separate MediaProcessors for: | |
87 | ||
88 | - initial_processing: the intial processing of a media | |
89 | - gen_thumb: generate a thumbnail | |
90 | - resize: resize an image | |
91 | - transcode: transcode a video | |
92 | ||
58350141 | 93 | ... etc. |
274a0f67 CAW |
94 | |
95 | Some information on producing a new MediaProcessor for your media type: | |
96 | ||
97 | - You *must* supply a name attribute. This must be a class level | |
98 | attribute, and a string. This will be used to determine the | |
99 | subcommand of your process | |
100 | - It's recommended that you supply a class level description | |
101 | attribute. | |
102 | - Supply a media_is_eligible classmethod. This will be used to | |
103 | determine whether or not a media entry is eligible to use this | |
104 | processor type. See the method documentation for details. | |
105 | - To give "./bin/gmg reprocess run" abilities to this media type, | |
106 | supply both gnerate_parser and parser_to_request classmethods. | |
107 | - The process method will be what actually processes your media. | |
e6bd03d4 | 108 | """ |
14565fb7 CAW |
109 | # You MUST override this in the child MediaProcessor! |
110 | name = None | |
111 | ||
274a0f67 CAW |
112 | # Optional, but will be used in various places to describe the |
113 | # action this MediaProcessor provides | |
114 | description = None | |
115 | ||
5fd239fa | 116 | def __init__(self, manager, entry): |
14565fb7 | 117 | self.manager = manager |
93b14fc3 | 118 | self.entry = entry |
5fd239fa | 119 | self.entry_orig_state = entry.state |
14565fb7 | 120 | |
e4bdc909 | 121 | # Should be initialized at time of processing, at least |
93b14fc3 | 122 | self.workbench = None |
93b14fc3 | 123 | |
55cfa340 CAW |
124 | def __enter__(self): |
125 | self.workbench = mgg.workbench_manager.create() | |
7a85bf98 | 126 | return self |
55cfa340 CAW |
127 | |
128 | def __exit__(self, *args): | |
129 | self.workbench.destroy() | |
130 | self.workbench = None | |
93b14fc3 | 131 | |
e4bdc909 | 132 | # @with_workbench |
274a0f67 | 133 | def process(self, **kwargs): |
93b14fc3 | 134 | """ |
274a0f67 | 135 | Actually process this media entry. |
93b14fc3 | 136 | """ |
14565fb7 CAW |
137 | raise NotImplementedError |
138 | ||
274a0f67 | 139 | @classmethod |
7584080b | 140 | def media_is_eligible(cls, entry=None, state=None): |
14565fb7 CAW |
141 | raise NotImplementedError |
142 | ||
274a0f67 CAW |
143 | ############################### |
144 | # Command line interface things | |
145 | ############################### | |
146 | ||
147 | @classmethod | |
55a10fef | 148 | def generate_parser(cls): |
14565fb7 CAW |
149 | raise NotImplementedError |
150 | ||
274a0f67 | 151 | @classmethod |
d1e9913b | 152 | def args_to_request(cls, args): |
274a0f67 CAW |
153 | raise NotImplementedError |
154 | ||
155 | ########################################## | |
156 | # THE FUTURE: web interface things here :) | |
157 | ########################################## | |
158 | ||
5fd239fa CAW |
159 | ##################### |
160 | # Some common "steps" | |
161 | ##################### | |
715ea495 | 162 | |
93b14fc3 | 163 | def delete_queue_file(self): |
8ec87dc3 E |
164 | # Remove queued media file from storage and database. |
165 | # queued_filepath is in the task_id directory which should | |
166 | # be removed too, but fail if the directory is not empty to be on | |
167 | # the super-safe side. | |
93b14fc3 | 168 | queued_filepath = self.entry.queued_media_file |
882779f5 RE |
169 | if queued_filepath: |
170 | mgg.queue_store.delete_file(queued_filepath) # rm file | |
171 | mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir | |
172 | self.entry.queued_media_file = [] | |
5fd239fa | 173 | |
14565fb7 | 174 | |
4ba5bdd9 CAW |
175 | class ProcessingKeyError(Exception): pass |
176 | class ProcessorDoesNotExist(ProcessingKeyError): pass | |
177 | class ProcessorNotEligible(ProcessingKeyError): pass | |
4e601368 RE |
178 | class ProcessingManagerDoesNotExist(ProcessingKeyError): pass |
179 | ||
4ba5bdd9 CAW |
180 | |
181 | ||
14565fb7 | 182 | class ProcessingManager(object): |
e4bdc909 | 183 | """Manages all the processing actions available for a media type |
14565fb7 | 184 | |
e4bdc909 CAW |
185 | Specific processing actions, MediaProcessor subclasses, are added |
186 | to the ProcessingManager. | |
187 | """ | |
188 | def __init__(self): | |
14565fb7 | 189 | # Dict of all MediaProcessors of this media type |
85ead8ac | 190 | self.processors = OrderedDict() |
14565fb7 CAW |
191 | |
192 | def add_processor(self, processor): | |
193 | """ | |
194 | Add a processor class to this media type | |
195 | """ | |
196 | name = processor.name | |
197 | if name is None: | |
198 | raise AttributeError("Processor class's .name attribute not set") | |
58350141 | 199 | |
14565fb7 CAW |
200 | self.processors[name] = processor |
201 | ||
e4bdc909 | 202 | def list_eligible_processors(self, entry): |
14565fb7 CAW |
203 | """ |
204 | List all processors that this media entry is eligible to be processed | |
205 | for. | |
206 | """ | |
207 | return [ | |
208 | processor | |
85ead8ac | 209 | for processor in self.processors.values() |
7584080b RE |
210 | if processor.media_is_eligible(entry=entry)] |
211 | ||
212 | def list_all_processors_by_state(self, state): | |
213 | """ | |
214 | List all processors that this media state is eligible to be processed | |
215 | for. | |
216 | """ | |
217 | return [ | |
218 | processor | |
219 | for processor in self.processors.values() | |
220 | if processor.media_is_eligible(state=state)] | |
221 | ||
e4bdc909 | 222 | |
85ead8ac CAW |
223 | def list_all_processors(self): |
224 | return self.processors.values() | |
225 | ||
e4bdc909 CAW |
226 | def gen_process_request_via_cli(self, subparser): |
227 | # Got to figure out what actually goes here before I can write this properly | |
228 | pass | |
14565fb7 | 229 | |
4ba5bdd9 CAW |
230 | def get_processor(self, key, entry=None): |
231 | """ | |
232 | Get the processor with this key. | |
233 | ||
234 | If entry supplied, make sure this entry is actually compatible; | |
235 | otherwise raise error. | |
236 | """ | |
237 | try: | |
238 | processor = self.processors[key] | |
239 | except KeyError: | |
98d1fa3b CAW |
240 | import pdb |
241 | pdb.set_trace() | |
4ba5bdd9 CAW |
242 | raise ProcessorDoesNotExist( |
243 | "'%s' processor does not exist for this media type" % key) | |
244 | ||
245 | if entry and not processor.media_is_eligible(entry): | |
246 | raise ProcessorNotEligible( | |
247 | "This entry is not eligible for processor with name '%s'" % key) | |
248 | ||
249 | return processor | |
250 | ||
14565fb7 | 251 | |
d1e9913b CAW |
252 | def request_from_args(args, which_args): |
253 | """ | |
254 | Generate a request from the values of some argparse parsed args | |
255 | """ | |
256 | request = {} | |
257 | for arg in which_args: | |
258 | request[arg] = getattr(args, arg) | |
259 | ||
260 | return request | |
261 | ||
262 | ||
77ea4c9b CAW |
263 | class MediaEntryNotFound(Exception): pass |
264 | ||
265 | ||
55cfa340 | 266 | def get_processing_manager_for_type(media_type): |
77ea4c9b CAW |
267 | """ |
268 | Get the appropriate media manager for this type | |
269 | """ | |
270 | manager_class = hook_handle(('reprocess_manager', media_type)) | |
4e601368 RE |
271 | if not manager_class: |
272 | raise ProcessingManagerDoesNotExist( | |
273 | "A processing manager does not exist for {0}".format(media_type)) | |
77ea4c9b CAW |
274 | manager = manager_class() |
275 | ||
276 | return manager | |
277 | ||
278 | ||
55cfa340 | 279 | def get_entry_and_processing_manager(media_id): |
77ea4c9b CAW |
280 | """ |
281 | Get a MediaEntry, its media type, and its manager all in one go. | |
282 | ||
283 | Returns a tuple of: `(entry, media_type, media_manager)` | |
284 | """ | |
285 | entry = MediaEntry.query.filter_by(id=media_id).first() | |
286 | if entry is None: | |
287 | raise MediaEntryNotFound("Can't find media with id '%s'" % media_id) | |
288 | ||
55cfa340 | 289 | manager = get_processing_manager_for_type(entry.media_type) |
77ea4c9b CAW |
290 | |
291 | return entry, manager | |
93b14fc3 E |
292 | |
293 | ||
6788b412 | 294 | def mark_entry_failed(entry_id, exc): |
2e5ea6b9 CAW |
295 | """ |
296 | Mark a media entry as having failed in its conversion. | |
297 | ||
243c3843 NY |
298 | Uses the exception that was raised to mark more information. If |
299 | the exception is a derivative of BaseProcessingFail then we can | |
300 | store extra information that can be useful for users telling them | |
301 | why their media failed to process. | |
2e5ea6b9 CAW |
302 | |
303 | Args: | |
304 | - entry_id: The id of the media entry | |
305 | ||
306 | """ | |
6788b412 CAW |
307 | # Was this a BaseProcessingFail? In other words, was this a |
308 | # type of error that we know how to handle? | |
309 | if isinstance(exc, BaseProcessingFail): | |
310 | # Looks like yes, so record information about that failure and any | |
311 | # metadata the user might have supplied. | |
82cd9683 | 312 | atomic_update(mgg.database.MediaEntry, |
5c2b8486 | 313 | {'id': entry_id}, |
82cd9683 | 314 | {u'state': u'failed', |
5bd0adeb | 315 | u'fail_error': unicode(exc.exception_path), |
82cd9683 | 316 | u'fail_metadata': exc.metadata}) |
6788b412 | 317 | else: |
baae1578 | 318 | _log.warn("No idea what happened here, but it failed: %r", exc) |
6788b412 CAW |
319 | # Looks like no, so just mark it as failed and don't record a |
320 | # failure_error (we'll assume it wasn't handled) and don't record | |
321 | # metadata (in fact overwrite it if somehow it had previous info | |
322 | # here) | |
82cd9683 | 323 | atomic_update(mgg.database.MediaEntry, |
5c2b8486 | 324 | {'id': entry_id}, |
82cd9683 E |
325 | {u'state': u'failed', |
326 | u'fail_error': None, | |
327 | u'fail_metadata': {}}) | |
4a477e24 CAW |
328 | |
329 | ||
1cefccc7 | 330 | def get_process_filename(entry, workbench, acceptable_files): |
eb372949 | 331 | """ |
1cefccc7 RE |
332 | Try and get the queued file if available, otherwise return the first file |
333 | in the acceptable_files that we have. | |
eb372949 | 334 | |
1cefccc7 | 335 | If no acceptable_files, raise ProcessFileNotFound |
eb372949 CAW |
336 | """ |
337 | if entry.queued_media_file: | |
1cefccc7 | 338 | filepath = entry.queued_media_file |
eb372949 CAW |
339 | storage = mgg.queue_store |
340 | else: | |
1cefccc7 RE |
341 | for keyname in acceptable_files: |
342 | if entry.media_files.get(keyname): | |
343 | filepath = entry.media_files[keyname] | |
344 | storage = mgg.public_store | |
345 | break | |
346 | ||
347 | if not filepath: | |
348 | raise ProcessFileNotFound() | |
eb372949 | 349 | |
1cefccc7 RE |
350 | filename = workbench.localized_file( |
351 | storage, filepath, | |
eb372949 CAW |
352 | 'source') |
353 | ||
1cefccc7 RE |
354 | if not os.path.exists(filename): |
355 | raise ProcessFileNotFound() | |
356 | ||
357 | return filename | |
eb372949 CAW |
358 | |
359 | ||
fb56676b RE |
360 | def store_public(entry, keyname, local_file, target_name=None, |
361 | delete_if_exists=True): | |
5fd239fa CAW |
362 | if target_name is None: |
363 | target_name = os.path.basename(local_file) | |
364 | target_filepath = create_pub_filepath(entry, target_name) | |
79f84d7e | 365 | |
5fd239fa CAW |
366 | if keyname in entry.media_files: |
367 | _log.warn("store_public: keyname %r already used for file %r, " | |
368 | "replacing with %r", keyname, | |
369 | entry.media_files[keyname], target_filepath) | |
fb56676b RE |
370 | if delete_if_exists: |
371 | mgg.public_store.delete_file(entry.media_files[keyname]) | |
79f84d7e RE |
372 | |
373 | try: | |
374 | mgg.public_store.copy_local_to_storage(local_file, target_filepath) | |
375 | except: | |
376 | raise PublicStoreFail(keyname=keyname) | |
377 | ||
378 | # raise an error if the file failed to copy | |
379 | copied_filepath = mgg.public_store.get_local_path(target_filepath) | |
380 | if not os.path.exists(copied_filepath): | |
381 | raise PublicStoreFail(keyname=keyname) | |
382 | ||
5fd239fa CAW |
383 | entry.media_files[keyname] = target_filepath |
384 | ||
385 | ||
386 | def copy_original(entry, orig_filename, target_name, keyname=u"original"): | |
387 | store_public(entry, keyname, orig_filename, target_name) | |
388 | ||
389 | ||
8e5f9746 JW |
390 | class BaseProcessingFail(Exception): |
391 | """ | |
392 | Base exception that all other processing failure messages should | |
393 | subclass from. | |
394 | ||
395 | You shouldn't call this itself; instead you should subclass it | |
2392fbc0 | 396 | and provide the exception_path and general_message applicable to |
8e5f9746 JW |
397 | this error. |
398 | """ | |
399 | general_message = u'' | |
400 | ||
401 | @property | |
402 | def exception_path(self): | |
403 | return u"%s:%s" % ( | |
404 | self.__class__.__module__, self.__class__.__name__) | |
405 | ||
406 | def __init__(self, **metadata): | |
407 | self.metadata = metadata or {} | |
408 | ||
8e5f9746 | 409 | class BadMediaFail(BaseProcessingFail): |
4a477e24 | 410 | """ |
8e5f9746 JW |
411 | Error that should be raised when an inappropriate file was given |
412 | for the media type specified. | |
4a477e24 | 413 | """ |
8e5f9746 | 414 | general_message = _(u'Invalid file given for media type.') |
79f84d7e RE |
415 | |
416 | ||
417 | class PublicStoreFail(BaseProcessingFail): | |
418 | """ | |
419 | Error that should be raised when copying to public store fails | |
420 | """ | |
421 | general_message = _('Copying to public storage failed.') | |
1cefccc7 RE |
422 | |
423 | ||
424 | class ProcessFileNotFound(BaseProcessingFail): | |
425 | """ | |
426 | Error that should be raised when an acceptable file for processing | |
427 | is not found. | |
428 | """ | |
429 | general_message = _(u'An acceptable processing file was not found') |