mediagoblin/tools/text.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import wtforms
  18 import markdown
  19 from lxml.html.clean import Cleaner
  20
  21 from mediagoblin import mg_globals
  22 from mediagoblin.tools import url
  23
  24
  25 # A super strict version of the lxml.html cleaner class
  26 HTML_CLEANER = Cleaner(
  27     scripts=True,
  28     javascript=True,
  29     comments=True,
  30     style=True,
  31     links=True,
  32     page_structure=True,
  33     processing_instructions=True,
  34     embedded=True,
  35     frames=True,
  36     forms=True,
  37     annoying_tags=True,
  38     allow_tags=[
  39         'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br'],
  40     remove_unknown_tags=False, # can't be used with allow_tags
  41     safe_attrs_only=True,
  42     add_nofollow=True, # for now
  43     host_whitelist=(),
  44     whitelist_tags=set([]))
  45
  46
  47 def clean_html(html):
  48     # clean_html barfs on an empty string
  49     if not html:
  50         return u''
  51
  52     return HTML_CLEANER.clean_html(html)
  53
  54
  55 def convert_to_tag_list_of_dicts(tag_string):
  56     """
  57     Filter input from incoming string containing user tags,
  58
  59     Strips trailing, leading, and internal whitespace, and also converts
  60     the "tags" text into an array of tags
  61     """
  62     taglist = []
  63     if tag_string:
  64
  65         # Strip out internal, trailing, and leading whitespace
  66         stripped_tag_string = u' '.join(tag_string.strip().split())
  67
  68         # Split the tag string into a list of tags
  69         for tag in stripped_tag_string.split(
  70                                        mg_globals.app_config['tags_delimiter']):
  71
  72             # Ignore empty or duplicate tags
  73             if tag.strip() and tag.strip() not in [t['name'] for t in taglist]:
  74
  75                 taglist.append({'name': tag.strip(),
  76                                 'slug': url.slugify(tag.strip())})
  77     return taglist
  78
  79
  80 def media_tags_as_string(media_entry_tags):
  81     """
  82     Generate a string from a media item's tags, stored as a list of dicts
  83
  84     This is the opposite of convert_to_tag_list_of_dicts
  85     """
  86     media_tag_string = ''
  87     if media_entry_tags:
  88         media_tag_string = mg_globals.app_config['tags_delimiter'].join(
  89                                       [tag['name'] for tag in media_entry_tags])
  90     return media_tag_string
  91
  92
  93 TOO_LONG_TAG_WARNING = \
  94     u'Tags must be shorter than %s characters.  Tags that are too long: %s'
  95
  96
  97 def tag_length_validator(form, field):
  98     """
  99     Make sure tags do not exceed the maximum tag length.
 100     """
 101     tags = convert_to_tag_list_of_dicts(field.data)
 102     too_long_tags = [
 103         tag['name'] for tag in tags
 104         if len(tag['name']) > mg_globals.app_config['tags_max_length']]
 105
 106     if too_long_tags:
 107         raise wtforms.ValidationError(
 108             TOO_LONG_TAG_WARNING % (mg_globals.app_config['tags_max_length'], \
 109                                     ', '.join(too_long_tags)))
 110
 111
 112 MARKDOWN_INSTANCE = markdown.Markdown(safe_mode='escape')
 113
 114
 115 def cleaned_markdown_conversion(text):
 116     """
 117     Take a block of text, run it through MarkDown, and clean its HTML.
 118     """
 119     # Markdown will do nothing with and clean_html can do nothing with
 120     # an empty string :)
 121     if not text:
 122         return u''
 123
 124     return clean_html(MARKDOWN_INSTANCE.convert(text))