From 1cefccc7554a5df4c9bb126ef3b80b53f9e41cd7 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 15 Aug 2013 08:54:09 -0700 Subject: [PATCH] refactor get_orig_filename to return an acceptable filename to the processor. If there is an original video file and we skip transcoding, delete the webm_640 file --- mediagoblin/media_types/ascii/processing.py | 18 +++++---- mediagoblin/media_types/audio/processing.py | 17 +++++---- mediagoblin/media_types/image/processing.py | 21 ++++++----- mediagoblin/media_types/pdf/processing.py | 20 +++++----- mediagoblin/media_types/stl/processing.py | 17 +++++---- mediagoblin/media_types/video/processing.py | 25 ++++++++----- mediagoblin/processing/__init__.py | 41 ++++++++++++++------- 7 files changed, 94 insertions(+), 65 deletions(-) diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index ab89f4ad..9b6b3ad4 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -26,7 +26,7 @@ from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( create_pub_filepath, FilenameBuilder, MediaProcessor, ProcessingManager, - get_orig_filename, copy_original, + get_process_filename, copy_original, store_public, request_from_args) from mediagoblin.media_types.ascii import asciitoimage @@ -52,6 +52,8 @@ class CommonAsciiProcessor(MediaProcessor): """ Provides a base for various ascii processing steps """ + acceptable_files = ['original', 'unicode'] + def common_setup(self): self.ascii_config = mgg.global_config[ 'media_type:mediagoblin.media_types.ascii'] @@ -61,16 +63,16 @@ class CommonAsciiProcessor(MediaProcessor): self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) self.charset = None def copy_original(self): copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _detect_charset(self, orig_file): @@ -91,7 +93,7 @@ class CommonAsciiProcessor(MediaProcessor): orig_file.seek(0) def store_unicode_file(self): - with file(self.orig_filename, 'rb') as orig_file: + with file(self.process_filename, 'rb') as orig_file: self._detect_charset(orig_file) unicode_filepath = create_pub_filepath(self.entry, 'ascii-portable.txt') @@ -110,7 +112,7 @@ class CommonAsciiProcessor(MediaProcessor): self.entry.media_files['unicode'] = unicode_filepath def generate_thumb(self, font=None, thumb_size=None): - with file(self.orig_filename, 'rb') as orig_file: + with file(self.process_filename, 'rb') as orig_file: # If no font kwarg, check config if not font: font = self.ascii_config.get('thumbnail_font', None) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 77647d12..8f1ddcbd 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -22,7 +22,7 @@ from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( BadMediaFail, FilenameBuilder, ProgressCallback, MediaProcessor, ProcessingManager, - request_from_args, get_orig_filename, + request_from_args, get_process_filename, store_public, copy_original) from mediagoblin.media_types.audio.transcoders import ( @@ -52,6 +52,7 @@ class CommonAudioProcessor(MediaProcessor): """ Provides a base for various audio processing steps """ + acceptable_files = ['original', 'webm_audio'] def common_setup(self): """ @@ -61,10 +62,10 @@ class CommonAudioProcessor(MediaProcessor): self.audio_config = mgg \ .global_config['media_type:mediagoblin.media_types.audio'] - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer() @@ -72,7 +73,7 @@ class CommonAudioProcessor(MediaProcessor): def copy_original(self): if self.audio_config['keep_original']: copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def transcode(self, quality=None): @@ -85,7 +86,7 @@ class CommonAudioProcessor(MediaProcessor): '{basename}{ext}')) self.transcoder.transcode( - self.orig_filename, + self.process_filename, webm_audio_tmp, quality=quality, progress_callback=progress_callback) @@ -107,7 +108,7 @@ class CommonAudioProcessor(MediaProcessor): _log.info('Creating OGG source for spectrogram') self.transcoder.transcode( - self.orig_filename, + self.process_filename, wav_tmp, mux_string='vorbisenc quality={0} ! oggmux'.format( self.audio_config['quality'])) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 81b4d449..eaa19261 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -26,7 +26,7 @@ from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( BadMediaFail, FilenameBuilder, MediaProcessor, ProcessingManager, - request_from_args, get_orig_filename, + request_from_args, get_process_filename, store_public, copy_original) from mediagoblin.tools.exif import exif_fix_image_orientation, \ extract_exif, clean_exif, get_gps_data, get_useful, \ @@ -131,6 +131,9 @@ class CommonImageProcessor(MediaProcessor): """ Provides a base for various media processing steps """ + # list of acceptable file keys in order of prefrence for reprocessing + acceptable_files = ['original', 'medium'] + def common_setup(self): """ Set up the workbench directory and pull down the original file @@ -144,13 +147,13 @@ class CommonImageProcessor(MediaProcessor): self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction - self.exif_tags = extract_exif(self.orig_filename) + self.exif_tags = extract_exif(self.process_filename) def generate_medium_if_applicable(self, size=None, quality=None, filter=None): @@ -159,7 +162,7 @@ class CommonImageProcessor(MediaProcessor): if not filter: filter = self.image_config['resize_filter'] - resize_tool(self.entry, False, 'medium', self.orig_filename, + resize_tool(self.entry, False, 'medium', self.process_filename, self.name_builder.fill('{basename}.medium{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) @@ -170,14 +173,14 @@ class CommonImageProcessor(MediaProcessor): if not filter: filter = self.image_config['resize_filter'] - resize_tool(self.entry, True, 'thumb', self.orig_filename, + resize_tool(self.entry, True, 'thumb', self.process_filename, self.name_builder.fill('{basename}.thumbnail{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def copy_original(self): copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def extract_metadata(self): diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index 19ab54b2..6ef95a72 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -23,7 +23,7 @@ from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, BadMediaFail, MediaProcessor, ProcessingManager, - request_from_args, get_orig_filename, + request_from_args, get_process_filename, store_public, copy_original) from mediagoblin.tools.translate import fake_ugettext_passthrough as _ @@ -239,20 +239,22 @@ class CommonPdfProcessor(MediaProcessor): """ Provides a base for various pdf processing steps """ + acceptable_files = ['original', 'pdf'] + def common_setup(self): """ Set up common pdf processing steps """ - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename() def _set_pdf_filename(self): if self.name_builder.ext == 'pdf': - self.pdf_filename = self.orig_filename + self.pdf_filename = self.process_filename elif self.entry.media_files.get('pdf'): self.pdf_filename = self.workbench.localized_file( mgg.public_store, self.entry.media_files['pdf']) @@ -261,7 +263,7 @@ class CommonPdfProcessor(MediaProcessor): def copy_original(self): copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def generate_thumb(self, thumb_size=None): @@ -290,11 +292,11 @@ class CommonPdfProcessor(MediaProcessor): """ Store the pdf. If the file is not a pdf, make it a pdf """ - tmp_pdf = self.orig_filename + tmp_pdf = self.process_filename unoconv = where('unoconv') Popen(executable=unoconv, - args=[unoconv, '-v', '-f', 'pdf', self.orig_filename]).wait() + args=[unoconv, '-v', '-f', 'pdf', self.process_filename]).wait() if not os.path.exists(tmp_pdf): _log.debug('unoconv failed to convert file to pdf') diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index fddb94a2..77d3d86e 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -25,7 +25,7 @@ from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, MediaProcessor, ProcessingManager, request_from_args, - get_orig_filename, store_public, + get_process_filename, store_public, copy_original) from mediagoblin.media_types.stl import model_loader @@ -83,12 +83,13 @@ class CommonStlProcessor(MediaProcessor): """ Provides a common base for various stl processing steps """ + acceptable_files = ['original'] def common_setup(self): - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) self._set_ext() self._set_model() @@ -107,7 +108,7 @@ class CommonStlProcessor(MediaProcessor): Attempt to parse the model file and divine some useful information about it. """ - with open(self.orig_filename, 'rb') as model_file: + with open(self.process_filename, 'rb') as model_file: self.model = model_loader.auto_detect(model_file, self.ext) def _set_greatest(self): @@ -117,14 +118,14 @@ class CommonStlProcessor(MediaProcessor): def copy_original(self): copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _snap(self, keyname, name, camera, size, project="ORTHO"): filename = self.name_builder.fill(name) workbench_path = self.workbench.joinpath(filename) shot = { - "model_path": self.orig_filename, + "model_path": self.process_filename, "model_ext": self.ext, "camera_coord": camera, "camera_focus": self.model.average, diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 4d7d9921..bb854ffb 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -24,7 +24,7 @@ from mediagoblin.processing import ( FilenameBuilder, BaseProcessingFail, ProgressCallback, MediaProcessor, ProcessingManager, request_from_args, - get_orig_filename, store_public, + get_process_filename, store_public, copy_original) from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ @@ -122,15 +122,16 @@ class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ + acceptable_files = ['original', 'webm_640'] def common_setup(self): self.video_config = mgg \ .global_config['media_type:mediagoblin.media_types.video'] - # Pull down and set up the original file - self.orig_filename = get_orig_filename( - self.entry, self.workbench) - self.name_builder = FilenameBuilder(self.orig_filename) + # Pull down and set up the processing file + self.process_filename = get_process_filename( + self.entry, self.workbench, self.acceptable_files) + self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False @@ -140,7 +141,7 @@ class CommonVideoProcessor(MediaProcessor): if not self.did_transcode or \ (self.video_config['keep_original'] and self.did_transcode): copy_original( - self.entry, self.orig_filename, + self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, @@ -161,7 +162,7 @@ class CommonVideoProcessor(MediaProcessor): vorbis_quality = self.video_config['vorbis_quality'] # Extract metadata and keep a record of it - metadata = self.transcoder.discover(self.orig_filename) + metadata = self.transcoder.discover(self.process_filename) store_metadata(self.entry, metadata) # Figure out whether or not we need to transcode this video or @@ -171,8 +172,14 @@ class CommonVideoProcessor(MediaProcessor): dst_dimensions = metadata['videowidth'], metadata['videoheight'] + # If there is an original and transcoded, delete the transcoded + # since it must be of lower quality then the original + if self.entry.media_files.get('original') and \ + self.entry.media_files.get('webm_640'): + self.entry.media_files['webm_640'].delete() + else: - self.transcoder.transcode(self.orig_filename, tmp_dst, + self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, @@ -206,7 +213,7 @@ class CommonVideoProcessor(MediaProcessor): mgg.global_config['media:thumb']['max_height']) transcoders.VideoThumbnailerMarkII( - self.orig_filename, + self.process_filename, tmp_thumb, thumb_size[0], thumb_size[1]) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index e2415fd5..746f4d8e 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -327,29 +327,34 @@ def mark_entry_failed(entry_id, exc): u'fail_metadata': {}}) -def get_orig_filename(entry, workbench): +def get_process_filename(entry, workbench, acceptable_files): """ - Get the a filename for the original, on local storage + Try and get the queued file if available, otherwise return the first file + in the acceptable_files that we have. - If the media entry has a queued_media_file, use that, otherwise - use the original. - - In the future, this will return the highest quality file available - if neither the original or queued file are available by checking - some ordered list of preferred keys. + If no acceptable_files, raise ProcessFileNotFound """ if entry.queued_media_file: - orig_filepath = entry.queued_media_file + filepath = entry.queued_media_file storage = mgg.queue_store else: - orig_filepath = entry.media_files['original'] - storage = mgg.public_store + for keyname in acceptable_files: + if entry.media_files.get(keyname): + filepath = entry.media_files[keyname] + storage = mgg.public_store + break + + if not filepath: + raise ProcessFileNotFound() - orig_filename = workbench.localized_file( - storage, orig_filepath, + filename = workbench.localized_file( + storage, filepath, 'source') - return orig_filename + if not os.path.exists(filename): + raise ProcessFileNotFound() + + return filename def store_public(entry, keyname, local_file, target_name=None, @@ -415,3 +420,11 @@ class PublicStoreFail(BaseProcessingFail): Error that should be raised when copying to public store fails """ general_message = _('Copying to public storage failed.') + + +class ProcessFileNotFound(BaseProcessingFail): + """ + Error that should be raised when an acceptable file for processing + is not found. + """ + general_message = _(u'An acceptable processing file was not found') -- 2.25.1