mediagoblin/media_types/ascii/processing.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 import argparse
  17 import chardet
  18 import os
  19 try:
  20     from PIL import Image
  21 except ImportError:
  22     import Image
  23 import logging
  24
  25 from mediagoblin import mg_globals as mgg
  26 from mediagoblin.processing import (
  27     create_pub_filepath, FilenameBuilder,
  28     MediaProcessor, ProcessingManager,
  29     get_process_filename, copy_original,
  30     store_public, request_from_args)
  31 from mediagoblin.media_types.ascii import asciitoimage
  32
  33 _log = logging.getLogger(__name__)
  34
  35 SUPPORTED_EXTENSIONS = ['txt', 'asc', 'nfo']
  36 MEDIA_TYPE = 'mediagoblin.media_types.ascii'
  37
  38
  39 def sniff_handler(media_file, **kw):
  40     _log.info('Sniffing {0}'.format(MEDIA_TYPE))
  41     if kw.get('media') is not None:
  42         name, ext = os.path.splitext(kw['media'].filename)
  43         clean_ext = ext[1:].lower()
  44
  45         if clean_ext in SUPPORTED_EXTENSIONS:
  46             return MEDIA_TYPE
  47
  48     return None
  49
  50
  51 class CommonAsciiProcessor(MediaProcessor):
  52     """
  53     Provides a base for various ascii processing steps
  54     """
  55     acceptable_files = ['original', 'unicode']
  56
  57     def common_setup(self):
  58         self.ascii_config = mgg.global_config['plugins'][
  59             'mediagoblin.media_types.ascii']
  60
  61          # Conversions subdirectory to avoid collisions
  62         self.conversions_subdir = os.path.join(
  63             self.workbench.dir, 'conversions')
  64         os.mkdir(self.conversions_subdir)
  65
  66         # Pull down and set up the processing file
  67         self.process_filename = get_process_filename(
  68             self.entry, self.workbench, self.acceptable_files)
  69         self.name_builder = FilenameBuilder(self.process_filename)
  70
  71         self.charset = None
  72
  73     def copy_original(self):
  74         copy_original(
  75             self.entry, self.process_filename,
  76             self.name_builder.fill('{basename}{ext}'))
  77
  78     def _detect_charset(self, orig_file):
  79         d_charset = chardet.detect(orig_file.read())
  80
  81         # Only select a non-utf-8 charset if chardet is *really* sure
  82         # Tested with "Feli\x0109an superjaron", which was detected
  83         if d_charset['confidence'] < 0.9:
  84             self.charset = 'utf-8'
  85         else:
  86             self.charset = d_charset['encoding']
  87
  88         _log.info('Charset detected: {0}\nWill interpret as: {1}'.format(
  89                   d_charset,
  90                   self.charset))
  91
  92         # Rewind the file
  93         orig_file.seek(0)
  94
  95     def store_unicode_file(self):
  96         with file(self.process_filename, 'rb') as orig_file:
  97             self._detect_charset(orig_file)
  98             unicode_filepath = create_pub_filepath(self.entry,
  99                                                    'ascii-portable.txt')
 100
 101             with mgg.public_store.get_file(unicode_filepath, 'wb') \
 102                     as unicode_file:
 103                 # Decode the original file from its detected charset (or UTF8)
 104                 # Encode the unicode instance to ASCII and replace any
 105                 # non-ASCII with an HTML entity (&#
 106                 unicode_file.write(
 107                     unicode(orig_file.read().decode(
 108                             self.charset)).encode(
 109                                 'ascii',
 110                                 'xmlcharrefreplace'))
 111
 112         self.entry.media_files['unicode'] = unicode_filepath
 113
 114     def generate_thumb(self, font=None, thumb_size=None):
 115         with file(self.process_filename, 'rb') as orig_file:
 116             # If no font kwarg, check config
 117             if not font:
 118                 font = self.ascii_config.get('thumbnail_font', None)
 119             if not thumb_size:
 120                 thumb_size = (mgg.global_config['media:thumb']['max_width'],
 121                               mgg.global_config['media:thumb']['max_height'])
 122
 123             if self._skip_resizing(font, thumb_size):
 124                 return
 125
 126             tmp_thumb = os.path.join(
 127                 self.conversions_subdir,
 128                 self.name_builder.fill('{basename}.thumbnail.png'))
 129
 130             ascii_converter_args = {}
 131
 132             # If there is a font from either the config or kwarg, update
 133             # ascii_converter_args
 134             if font:
 135                 ascii_converter_args.update(
 136                     {'font': self.ascii_config['thumbnail_font']})
 137
 138             converter = asciitoimage.AsciiToImage(
 139                 **ascii_converter_args)
 140
 141             thumb = converter._create_image(
 142                 orig_file.read())
 143
 144             with file(tmp_thumb, 'w') as thumb_file:
 145                 thumb.thumbnail(
 146                     thumb_size,
 147                     Image.ANTIALIAS)
 148                 thumb.save(thumb_file)
 149
 150             thumb_info = {'font': font,
 151                           'width': thumb_size[0],
 152                           'height': thumb_size[1]}
 153
 154             self.entry.set_file_metadata('thumb', **thumb_info)
 155
 156             _log.debug('Copying local file to public storage')
 157             store_public(self.entry, 'thumb', tmp_thumb,
 158                          self.name_builder.fill('{basename}.thumbnail.jpg'))
 159
 160         def _skip_resizing(self, font, thumb_size):
 161             thumb_info = self.entry.get_file_metadata('thumb')
 162
 163             if not thumb_info:
 164                 return False
 165
 166             skip = True
 167
 168             if thumb_info.get('font') != font:
 169                 skip = False
 170             elif thumb_info.get('width') != thumb_size[0]:
 171                 skip = False
 172             elif thumb_info.get('height') != thumb_size[1]:
 173                 skip = False
 174
 175             return skip
 176
 177
 178 class InitialProcessor(CommonAsciiProcessor):
 179     """
 180     Initial processing step for new ascii media
 181     """
 182     name = "initial"
 183     description = "Initial processing"
 184
 185     @classmethod
 186     def media_is_eligible(cls, entry=None, state=None):
 187         if not state:
 188             state = entry.state
 189         return state in (
 190             "unprocessed", "failed")
 191
 192     @classmethod
 193     def generate_parser(cls):
 194         parser = argparse.ArgumentParser(
 195             description=cls.description,
 196             prog=cls.name)
 197
 198         parser.add_argument(
 199             '--thumb_size',
 200             nargs=2,
 201             metavar=('max_width', 'max_width'),
 202             type=int)
 203
 204         parser.add_argument(
 205             '--font',
 206             help='the thumbnail font')
 207
 208         return parser
 209
 210     @classmethod
 211     def args_to_request(cls, args):
 212         return request_from_args(
 213             args, ['thumb_size', 'font'])
 214
 215     def process(self, thumb_size=None, font=None):
 216         self.common_setup()
 217         self.store_unicode_file()
 218         self.generate_thumb(thumb_size=thumb_size, font=font)
 219         self.copy_original()
 220         self.delete_queue_file()
 221
 222
 223 class Resizer(CommonAsciiProcessor):
 224     """
 225     Resizing process steps for processed media
 226     """
 227     name = 'resize'
 228     description = 'Resize thumbnail'
 229     thumb_size = 'thumb_size'
 230
 231     @classmethod
 232     def media_is_eligible(cls, entry=None, state=None):
 233         """
 234         Determine if this media type is eligible for processing
 235         """
 236         if not state:
 237             state = entry.state
 238         return state in 'processed'
 239
 240     @classmethod
 241     def generate_parser(cls):
 242         parser = argparse.ArgumentParser(
 243             description=cls.description,
 244             prog=cls.name)
 245
 246         parser.add_argument(
 247             '--thumb_size',
 248             nargs=2,
 249             metavar=('max_width', 'max_height'),
 250             type=int)
 251
 252         # Needed for gmg reprocess thumbs to work
 253         parser.add_argument(
 254             'file',
 255             nargs='?',
 256             default='thumb',
 257             choices=['thumb'])
 258
 259         return parser
 260
 261     @classmethod
 262     def args_to_request(cls, args):
 263         return request_from_args(
 264             args, ['thumb_size', 'file'])
 265
 266     def process(self, thumb_size=None, file=None):
 267         self.common_setup()
 268         self.generate_thumb(thumb_size=thumb_size)
 269
 270
 271 class AsciiProcessingManager(ProcessingManager):
 272     def __init__(self):
 273         super(self.__class__, self).__init__()
 274         self.add_processor(InitialProcessor)
 275         self.add_processor(Resizer)