From 33d5ac6c4de44aa8d98a8cf759c79351f5a59885 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 00:53:39 +0530 Subject: [PATCH] Simple hack to handle main workflow problem Remove redundunt workflow methods from the other media type's processing.py. Fixes #1 --- mediagoblin/media_types/ascii/processing.py | 5 ----- mediagoblin/media_types/audio/processing.py | 5 ----- mediagoblin/media_types/image/processing.py | 5 ----- mediagoblin/media_types/pdf/processing.py | 5 ----- .../media_types/raw_image/processing.py | 5 ----- mediagoblin/media_types/stl/processing.py | 5 ----- mediagoblin/media_types/video/processing.py | 21 +++++++------------ mediagoblin/processing/__init__.py | 5 ++--- mediagoblin/submit/lib.py | 6 +++++- 9 files changed, 15 insertions(+), 47 deletions(-) diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index c9b47fb5..00d04e63 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -273,8 +273,3 @@ class AsciiProcessingManager(ProcessingManager): super(AsciiProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 15d0b0a7..427309de 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -365,8 +365,3 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 7224a8fd..7ddf3f35 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,11 +431,6 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) - if __name__ == '__main__': import sys diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index e6e6e0a9..ac4bab6d 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -470,8 +470,3 @@ class PdfProcessingManager(ProcessingManager): super(PdfProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 4bfd9f3a..740ba2dd 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -80,8 +80,3 @@ class RawImageProcessingManager(ProcessingManager): super(RawImageProcessingManager, self).__init__() self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index cd3ffd8c..55764aeb 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -368,8 +368,3 @@ class StlProcessingManager(ProcessingManager): super(StlProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 586a0c85..e881636c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -30,7 +30,6 @@ from mediagoblin.processing import ( ProcessingManager, request_from_args, get_process_filename, store_public, copy_original, get_entry_and_processing_manager) -from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -543,7 +542,7 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): reprocess_info = reprocess_info or {} if 'vp8_quality' not in reprocess_info: @@ -556,25 +555,21 @@ class VideoProcessingManager(ProcessingManager): reprocess_info['thumb_size'] = None transcoding_tasks = group([ - main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), + main_task.signature(args=(entry.id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), + complimentary_task.signature(args=(entry.id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), - complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), + complimentary_task.signature(args=(entry.id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), ]) - cleanup_task = processing_cleanup.signature(args=(entry_id,), + cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) - """ - main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), - kwargs=reprocess_info, queue='default', - priority=5, immutable=True) - processing_cleanup.apply_async(args=(entry_id,), queue='default', immutable=True) - """ - chord(transcoding_tasks)(cleanup_task) + + # Not sure what to return since we are scheduling the task here itself + return 1 diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 98031bbc..7d407a36 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,12 +257,11 @@ class ProcessingManager(object): return processor - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): """ Returns the Celery command needed to proceed with media processing - *This method has to be implemented in all media types* """ - raise NotImplementedError + return None def request_from_args(args, which_args): diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index a6d564f1..b228dbd1 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -266,7 +266,11 @@ def run_process_media(entry, feed_url=None, entry, manager = get_entry_and_processing_manager(entry.id) try: - manager.workflow(entry.id, feed_url, reprocess_action, reprocess_info) + wf = manager.workflow(entry, feed_url, reprocess_action, reprocess_info) + if wf is None: + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that -- 2.25.1