mediagoblin/media_types/ascii/processing.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 import argparse
  17 import chardet
  18 import os
  19 try:
  20     from PIL import Image
  21 except ImportError:
  22     import Image
  23 import logging
  24
  25 from mediagoblin import mg_globals as mgg
  26 from mediagoblin.processing import (
  27     create_pub_filepath, FilenameBuilder,
  28     MediaProcessor, ProcessingManager,
  29     get_orig_filename, copy_original,
  30     store_public, request_from_args)
  31 from mediagoblin.media_types.ascii import asciitoimage
  32
  33 _log = logging.getLogger(__name__)
  34
  35 SUPPORTED_EXTENSIONS = ['txt', 'asc', 'nfo']
  36 MEDIA_TYPE = 'mediagoblin.media_types.ascii'
  37
  38
  39 def sniff_handler(media_file, **kw):
  40     _log.info('Sniffing {0}'.format(MEDIA_TYPE))
  41     if kw.get('media') is not None:
  42         name, ext = os.path.splitext(kw['media'].filename)
  43         clean_ext = ext[1:].lower()
  44
  45         if clean_ext in SUPPORTED_EXTENSIONS:
  46             return MEDIA_TYPE
  47
  48     return None
  49
  50
  51 class CommonAsciiProcessor(MediaProcessor):
  52     """
  53     Provides a base for various ascii processing steps
  54     """
  55     def common_setup(self):
  56         self.ascii_config = mgg.global_config[
  57             'media_type:mediagoblin.media_types.ascii']
  58
  59          # Conversions subdirectory to avoid collisions
  60         self.conversions_subdir = os.path.join(
  61             self.workbench.dir, 'convirsions')
  62         os.mkdir(self.conversions_subdir)
  63
  64         # Pull down and set up the original file
  65         self.orig_filename = get_orig_filename(
  66             self.entry, self.workbench)
  67         self.name_builder = FilenameBuilder(self.orig_filename)
  68
  69         self.charset = None
  70
  71     def copy_original(self):
  72         copy_original(
  73             self.entry, self.orig_filename,
  74             self.name_builder.fill('{basename}{ext}'))
  75
  76     def _detect_charset(self, orig_file):
  77         d_charset = chardet.detect(orig_file.read())
  78
  79         # Only select a non-utf-8 charset if chardet is *really* sure
  80         # Tested with "Feli\x0109an superjaron", which was detected
  81         if d_charset['confidence'] < 0.9:
  82             self.charset = 'utf-8'
  83         else:
  84             self.charset = d_charset['encoding']
  85
  86         _log.info('Charset detected: {0}\nWill interpret as: {1}'.format(
  87                   d_charset,
  88                   self.charset))
  89
  90         # Rewind the file
  91         orig_file.seek(0)
  92
  93     def store_unicode_file(self):
  94         with file(self.orig_filename, 'rb') as orig_file:
  95             self._detect_charset(orig_file)
  96             unicode_filepath = create_pub_filepath(self.entry,
  97                                                    'ascii-portable.txt')
  98
  99             with mgg.public_store.get_file(unicode_filepath, 'wb') \
 100                     as unicode_file:
 101                 # Decode the original file from its detected charset (or UTF8)
 102                 # Encode the unicode instance to ASCII and replace any
 103                 # non-ASCII with an HTML entity (&#
 104                 unicode_file.write(
 105                     unicode(orig_file.read().decode(
 106                             self.charset)).encode(
 107                                 'ascii',
 108                                 'xmlcharrefreplace'))
 109
 110         self.entry.media_files['unicode'] = unicode_filepath
 111
 112     def generate_thumb(self, font=None, thumb_size=None):
 113         with file(self.orig_filename, 'rb') as orig_file:
 114             # If no font kwarg, check config
 115             if not font:
 116                 font = self.ascii_config.get('thumbnail_font', None)
 117             if not thumb_size:
 118                 thumb_size = (mgg.global_config['media:thumb']['max_width'],
 119                               mgg.global_config['media:thumb']['max_height'])
 120
 121             tmp_thumb = os.path.join(
 122                 self.conversions_subdir,
 123                 self.name_builder.fill('{basename}.thumbnail.png'))
 124
 125             ascii_converter_args = {}
 126
 127             # If there is a font from either the config or kwarg, update
 128             # ascii_converter_args
 129             if font:
 130                 ascii_converter_args.update(
 131                     {'font': self.ascii_config['thumbnail_font']})
 132
 133             converter = asciitoimage.AsciiToImage(
 134                 **ascii_converter_args)
 135
 136             thumb = converter._create_image(
 137                 orig_file.read())
 138
 139             with file(tmp_thumb, 'w') as thumb_file:
 140                 thumb.thumbnail(
 141                     thumb_size,
 142                     Image.ANTIALIAS)
 143                 thumb.save(thumb_file)
 144
 145             _log.debug('Copying local file to public storage')
 146             store_public(self.entry, 'thumb', tmp_thumb,
 147                          self.name_builder.fill('{basename}.thumbnail.jpg'))
 148
 149
 150 class InitialProcessor(CommonAsciiProcessor):
 151     """
 152     Initial processing step for new ascii media
 153     """
 154     name = "initial"
 155     description = "Initial processing"
 156
 157     @classmethod
 158     def media_is_eligible(cls, entry=None, state=None):
 159         if not state:
 160             state = entry.state
 161         return state in (
 162             "unprocessed", "failed")
 163
 164     @classmethod
 165     def generate_parser(cls):
 166         parser = argparse.ArgumentParser(
 167             description=cls.description,
 168             prog=cls.name)
 169
 170         parser.add_argument(
 171             '--thumb_size',
 172             nargs=2,
 173             metavar=('max_width', 'max_width'),
 174             type=int)
 175
 176         parser.add_argument(
 177             '--font',
 178             help='the thumbnail font')
 179
 180         return parser
 181
 182     @classmethod
 183     def args_to_request(cls, args):
 184         return request_from_args(
 185             args, ['thumb_size', 'font'])
 186
 187     def process(self, thumb_size=None, font=None):
 188         self.common_setup()
 189         self.store_unicode_file()
 190         self.generate_thumb(thumb_size=thumb_size, font=font)
 191         self.copy_original()
 192         self.delete_queue_file()
 193
 194
 195 class Resizer(CommonAsciiProcessor):
 196     """
 197     Resizing process steps for processed media
 198     """
 199     name = 'resize'
 200     description = 'Resize thumbnail'
 201
 202     @classmethod
 203     def media_is_eligible(cls, entry=None, state=None):
 204         """
 205         Determine if this media type is eligible for processing
 206         """
 207         if not state:
 208             state = entry.state
 209         return state in 'processed'
 210
 211     @classmethod
 212     def generate_parser(cls):
 213         parser = argparse.ArgumentParser(
 214             description=cls.description,
 215             prog=cls.name)
 216
 217         parser.add_argument(
 218             '--thumb_size',
 219             nargs=2,
 220             metavar=('max_width', 'max_height'),
 221             type=int)
 222
 223         # Needed for gmg reprocess thumbs to work
 224         parser.add_argument(
 225             'file',
 226             nargs='?',
 227             default='thumb')
 228
 229         return parser
 230
 231     @classmethod
 232     def args_to_request(cls, args):
 233         return request_from_args(
 234             args, ['size', 'file'])
 235
 236     def process(self, thumb_size=None, file=None):
 237         self.common_setup()
 238         self.generate_thumb(thumb_size=thumb_size)
 239
 240
 241 class AsciiProcessingManager(ProcessingManager):
 242     def __init__(self):
 243         super(self.__class__, self).__init__()
 244         self.add_processor(InitialProcessor)