From 6471291575c97f03d129051dc3d2bef28b4d89f2 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Wed, 11 Jul 2012 00:36:42 +0200 Subject: [PATCH] Panel improvements - Added progress meter for video and audio media types. - Changed the __repr__ method of a MediaEntry to display a bit more useful explanation. - Added a new MediaEntry.state, 'processing', which means that the task is running the processor on the item currently. - Fixed some PEP8 issues in user_pages/views.py - Fixed the ATOM TAG URI to show the correct year. --- mediagoblin/db/sql/migrations.py | 14 +++++++- mediagoblin/db/sql/models.py | 8 +++++ mediagoblin/media_types/audio/processing.py | 8 +++-- mediagoblin/media_types/audio/transcoders.py | 2 +- mediagoblin/media_types/video/processing.py | 6 ++-- mediagoblin/media_types/video/transcoders.py | 2 +- mediagoblin/processing/__init__.py | 11 ++++++ mediagoblin/processing/task.py | 19 ++++++----- .../user_pages/processing_panel.html | 34 ++++++++++++++++--- mediagoblin/user_pages/views.py | 24 +++++++++---- 10 files changed, 101 insertions(+), 27 deletions(-) diff --git a/mediagoblin/db/sql/migrations.py b/mediagoblin/db/sql/migrations.py index d6b709b2..49798a54 100644 --- a/mediagoblin/db/sql/migrations.py +++ b/mediagoblin/db/sql/migrations.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from sqlalchemy import MetaData, Table, Column, Boolean +from sqlalchemy import MetaData, Table, Column, Boolean, SmallInteger from mediagoblin.db.sql.util import RegisterMigration @@ -47,3 +47,15 @@ def add_wants_notification_column(db_conn): default=True, nullable=True) col.create(users, populate_defaults=True) db_conn.commit() + + +@RegisterMigration(3, MIGRATIONS) +def add_transcoding_progress(db_conn): + metadata = MetaData(bind=db_conn.bind) + + media_entry = Table('core__media_entries', metadata, autoload=True, + autoload_with=db_conn.bind) + + col = Column('transcoding_progress', SmallInteger) + col.create(media_entry) + db_conn.commit() diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 9815fcf9..6ce4e06f 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -107,6 +107,8 @@ class MediaEntry(Base, MediaEntryMixin): fail_error = Column(Unicode) fail_metadata = Column(JSONEncoded) + transcoding_progress = Column(SmallInteger) + queued_media_file = Column(PathTupleWithSlashes) queued_task_id = Column(Unicode) @@ -209,6 +211,12 @@ class MediaEntry(Base, MediaEntryMixin): __import__(models_module) return sys.modules[models_module].DATA_MODEL + def __repr__(self): + return '<{classname} {id}: {title}>'.format( + classname=self.__class__.__name__, + id=self.id, + title=self.title) + class FileKeynames(Base): """ diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 558a37f0..18edbf1a 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -20,13 +20,14 @@ import os from mediagoblin import mg_globals as mgg from mediagoblin.processing import (create_pub_filepath, BadMediaFail, - FilenameBuilder) + FilenameBuilder, ProgressCallback) from mediagoblin.media_types.audio.transcoders import (AudioTranscoder, AudioThumbnailer) _log = logging.getLogger(__name__) + def sniff_handler(media_file, **kw): try: transcoder = AudioTranscoder() @@ -40,6 +41,7 @@ def sniff_handler(media_file, **kw): return False + def process_audio(entry): audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] @@ -72,11 +74,13 @@ def process_audio(entry): transcoder = AudioTranscoder() with tempfile.NamedTemporaryFile() as webm_audio_tmp: + progress_callback = ProgressCallback(entry) transcoder.transcode( queued_filename, webm_audio_tmp.name, - quality=audio_config['quality']) + quality=audio_config['quality'], + progress_callback=progress_callback) transcoder.discover(webm_audio_tmp.name) diff --git a/mediagoblin/media_types/audio/transcoders.py b/mediagoblin/media_types/audio/transcoders.py index be80aa0e..f3d49c30 100644 --- a/mediagoblin/media_types/audio/transcoders.py +++ b/mediagoblin/media_types/audio/transcoders.py @@ -206,7 +206,7 @@ class AudioTranscoder(object): data = dict(message.structure) if self.__on_progress: - self.__on_progress(data) + self.__on_progress(data.get('percent')) _log.info('{0}% done...'.format( data.get('percent'))) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 85e83352..abd14eed 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -19,7 +19,7 @@ import logging from mediagoblin import mg_globals as mgg from mediagoblin.processing import \ - create_pub_filepath, FilenameBuilder, BaseProcessingFail + create_pub_filepath, FilenameBuilder, BaseProcessingFail, ProgressCallback from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from . import transcoders @@ -78,11 +78,13 @@ def process_video(entry): with tmp_dst: # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square + progress_callback = ProgressCallback(entry) transcoder = transcoders.VideoTranscoder() transcoder.transcode(queued_filename, tmp_dst.name, vp8_quality=video_config['vp8_quality'], vp8_threads=video_config['vp8_threads'], - vorbis_quality=video_config['vorbis_quality']) + vorbis_quality=video_config['vorbis_quality'], + progress_callback=progress_callback) # Push transcoded video to public storage _log.debug('Saving medium...') diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index e03f721f..25846ffa 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -625,7 +625,7 @@ class VideoTranscoder: data = dict(message.structure) if self._progress_callback: - self._progress_callback(data) + self._progress_callback(data.get('percent')) _log.info('{percent}% done...'.format( percent=data.get('percent'))) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 85b61880..6b2d50e2 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -25,12 +25,23 @@ from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ _log = logging.getLogger(__name__) +class ProgressCallback(object): + def __init__(self, entry): + self.entry = entry + + def __call__(self, progress): + if progress: + self.entry.transcoding_progress = progress + self.entry.save() + + def create_pub_filepath(entry, filename): return mgg.public_store.get_unique_filepath( ['media_entries', unicode(entry._id), filename]) + class FilenameBuilder(object): """Easily slice and dice filenames. diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index af815362..58e36a11 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -44,20 +44,24 @@ class ProcessMedia(Task): entry = mgg.database.MediaEntry.one( {'_id': ObjectId(media_id)}) - _log.info('Running task {0} on media {1}: {2}'.format( - self.name, - entry._id, - entry.title)) - # Try to process, and handle expected errors. try: - #__import__(entry.media_type) manager = get_media_manager(entry.media_type) + + entry.state = u'processing' + entry.save() + _log.debug('Processing {0}'.format(entry)) + manager['processor'](entry) + + entry.state = u'processed' + entry.save() + except BaseProcessingFail as exc: mark_entry_failed(entry._id, exc) return + except ImportError as exc: _log.error( 'Entry {0} failed to process due to an import error: {1}'\ @@ -67,9 +71,6 @@ class ProcessMedia(Task): mark_entry_failed(entry._id, exc) - entry.state = u'processed' - entry.save() - def on_failure(self, exc, task_id, args, kwargs, einfo): """ If the processing failed we should mark that in the database. diff --git a/mediagoblin/templates/mediagoblin/user_pages/processing_panel.html b/mediagoblin/templates/mediagoblin/user_pages/processing_panel.html index e868456b..ac2fd44c 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/processing_panel.html +++ b/mediagoblin/templates/mediagoblin/user_pages/processing_panel.html @@ -33,19 +33,23 @@ ID Title When submitted - Status + Transcoding progress {% for media_entry in processing_entries %} {{ media_entry._id }} {{ media_entry.title }} {{ media_entry.created.strftime("%m-%d-%Y %I:%M %p") }} - + {% if media_entry.transcoding_progress %} + {{ media_entry.transcoding_progress }}% + {% else %} + Unknown + {% endif %} {% endfor %} {% else %} -

{% trans %}No media in-processing{% endtrans %}

+

{% trans %}No media in-processing{% endtrans %}

{% endif %} {% if failed_entries.count() %} @@ -74,5 +78,27 @@ {% endfor %} -{% endif %} +{% else %} +

{% trans %}No failed entries!{% endtrans %}

+{% endif %} +{% if processed_entries.count() %} +

{% trans %}Your last 10 successful uploads{% endtrans %}

+ + + + + + + + {% for entry in processed_entries %} + + + + + + {% endfor %} +
IDTitleSubmitted
{{ entry._id }}{{ entry.title }}{{ entry.created.strftime("%m-%d-%Y %I:%M %p") }}
+{% else %} +

{% trans %}No processed entries, yet!{% endtrans %}

+{% endif %} {% endblock %} diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 0e061c46..4c86aadc 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -16,6 +16,7 @@ from webob import exc import logging +import datetime from mediagoblin import messages, mg_globals from mediagoblin.db.util import DESCENDING, ObjectId @@ -37,6 +38,7 @@ from mediagoblin.media_types import get_media_manager _log = logging.getLogger(__name__) _log.setLevel(logging.DEBUG) + @uses_pagination def user_home(request, page): """'Homepage' of a User()""" @@ -251,10 +253,11 @@ def atom_feed(request): atomlinks = [{ 'href': request.urlgen( 'mediagoblin.user_pages.user_home', - qualified=True,user=request.matchdict['user']), + qualified=True, user=request.matchdict['user']), 'rel': 'alternate', 'type': 'text/html' - }]; + }] + if mg_globals.app_config["push_urls"]: for push_url in mg_globals.app_config["push_urls"]: atomlinks.append({ @@ -264,14 +267,16 @@ def atom_feed(request): feed = AtomFeed( "MediaGoblin: Feed for user '%s'" % request.matchdict['user'], feed_url=request.url, - id='tag:'+request.host+',2011:gallery.user-'+request.matchdict['user'], + id='tag:{host},{year}:gallery.user-{user}'.format( + host=request.host, + year=datetime.datetime.today().strftime('%Y'), + user=request.matchdict['user']), links=atomlinks) - for entry in cursor: feed.add(entry.get('title'), entry.description_html, - id=entry.url_for_self(request.urlgen,qualified=True), + id=entry.url_for_self(request.urlgen, qualified=True), content_type='html', author={ 'name': entry.get_uploader.username, @@ -323,17 +328,22 @@ def processing_panel(request): # Get media entries which are in-processing processing_entries = request.db.MediaEntry.find( {'uploader': user._id, - 'state': u'unprocessed'}).sort('created', DESCENDING) + 'state': u'processing'}).sort('created', DESCENDING) # Get media entries which have failed to process failed_entries = request.db.MediaEntry.find( {'uploader': user._id, 'state': u'failed'}).sort('created', DESCENDING) + processed_entries = request.db.MediaEntry.find( + {'uploader': user._id, + 'state': u'processed'}).sort('created', DESCENDING).limit(10) + # Render to response return render_to_response( request, 'mediagoblin/user_pages/processing_panel.html', {'user': user, 'processing_entries': processing_entries, - 'failed_entries': failed_entries}) + 'failed_entries': failed_entries, + 'processed_entries': processed_entries}) -- 2.25.1