mediagoblin/util.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011 Free Software Foundation, Inc
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from email.MIMEText import MIMEText
  18 import gettext
  19 import pkg_resources
  20 import smtplib
  21 import sys
  22 import re
  23 import urllib
  24 from math import ceil
  25 import copy
  26
  27 from babel.localedata import exists
  28 import jinja2
  29 import translitcodec
  30 from webob import Response, exc
  31 from lxml.html.clean import Cleaner
  32 import markdown
  33
  34 from mediagoblin import mg_globals
  35 from mediagoblin import messages
  36 from mediagoblin.db.util import ObjectId
  37
  38 TESTS_ENABLED = False
  39 def _activate_testing():
  40     """
  41     Call this to activate testing in util.py
  42     """
  43     global TESTS_ENABLED
  44     TESTS_ENABLED = True
  45
  46
  47 def clear_test_buckets():
  48     """
  49     We store some things for testing purposes that should be cleared
  50     when we want a "clean slate" of information for our next round of
  51     tests.  Call this function to wipe all that stuff clean.
  52
  53     Also wipes out some other things we might redefine during testing,
  54     like the jinja envs.
  55     """
  56     global SETUP_JINJA_ENVS
  57     SETUP_JINJA_ENVS = {}
  58
  59     global EMAIL_TEST_INBOX
  60     global EMAIL_TEST_MBOX_INBOX
  61     EMAIL_TEST_INBOX = []
  62     EMAIL_TEST_MBOX_INBOX = []
  63
  64     clear_test_template_context()
  65
  66
  67 def get_jinja_loader(user_template_path=None):
  68     """
  69     Set up the Jinja template loaders, possibly allowing for user
  70     overridden templates.
  71
  72     (In the future we may have another system for providing theming;
  73     for now this is good enough.)
  74     """
  75     if user_template_path:
  76         return jinja2.ChoiceLoader(
  77             [jinja2.FileSystemLoader(user_template_path),
  78              jinja2.PackageLoader('mediagoblin', 'templates')])
  79     else:
  80         return jinja2.PackageLoader('mediagoblin', 'templates')
  81
  82
  83 SETUP_JINJA_ENVS = {}
  84
  85
  86 def get_jinja_env(template_loader, locale):
  87     """
  88     Set up the Jinja environment,
  89
  90     (In the future we may have another system for providing theming;
  91     for now this is good enough.)
  92     """
  93     setup_gettext(locale)
  94
  95     # If we have a jinja environment set up with this locale, just
  96     # return that one.
  97     if SETUP_JINJA_ENVS.has_key(locale):
  98         return SETUP_JINJA_ENVS[locale]
  99
 100     template_env = jinja2.Environment(
 101         loader=template_loader, autoescape=True,
 102         extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape'])
 103
 104     template_env.install_gettext_callables(
 105         mg_globals.translations.gettext,
 106         mg_globals.translations.ngettext)
 107
 108     # All templates will know how to ...
 109     # ... fetch all waiting messages and remove them from the queue
 110     template_env.globals['fetch_messages'] = messages.fetch_messages
 111
 112     if exists(locale):
 113         SETUP_JINJA_ENVS[locale] = template_env
 114
 115     return template_env
 116
 117
 118 # We'll store context information here when doing unit tests
 119 TEMPLATE_TEST_CONTEXT = {}
 120
 121
 122 def render_template(request, template_path, context):
 123     """
 124     Render a template with context.
 125
 126     Always inserts the request into the context, so you don't have to.
 127     Also stores the context if we're doing unit tests.  Helpful!
 128     """
 129     template = request.template_env.get_template(
 130         template_path)
 131     context['request'] = request
 132     rendered = template.render(context)
 133
 134     if TESTS_ENABLED:
 135         TEMPLATE_TEST_CONTEXT[template_path] = context
 136
 137     return rendered
 138
 139
 140 def clear_test_template_context():
 141     global TEMPLATE_TEST_CONTEXT
 142     TEMPLATE_TEST_CONTEXT = {}
 143
 144
 145 def render_to_response(request, template, context):
 146     """Much like Django's shortcut.render()"""
 147     return Response(render_template(request, template, context))
 148
 149
 150 def redirect(request, *args, **kwargs):
 151     """Returns a HTTPFound(), takes a request and then urlgen params"""
 152     return exc.HTTPFound(location=request.urlgen(*args, **kwargs))
 153
 154
 155 def setup_user_in_request(request):
 156     """
 157     Examine a request and tack on a request.user parameter if that's
 158     appropriate.
 159     """
 160     if not request.session.has_key('user_id'):
 161         request.user = None
 162         return
 163
 164     user = None
 165     user = request.app.db.User.one(
 166         {'_id': ObjectId(request.session['user_id'])})
 167
 168     if not user:
 169         # Something's wrong... this user doesn't exist?  Invalidate
 170         # this session.
 171         request.session.invalidate()
 172
 173     request.user = user
 174
 175
 176 def import_component(import_string):
 177     """
 178     Import a module component defined by STRING.  Probably a method,
 179     class, or global variable.
 180
 181     Args:
 182      - import_string: a string that defines what to import.  Written
 183        in the format of "module1.module2:component"
 184     """
 185     module_name, func_name = import_string.split(':', 1)
 186     __import__(module_name)
 187     module = sys.modules[module_name]
 188     func = getattr(module, func_name)
 189     return func
 190
 191 _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')
 192
 193 def slugify(text, delim=u'-'):
 194     """
 195     Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/
 196     """
 197     result = []
 198     for word in _punct_re.split(text.lower()):
 199         word = word.encode('translit/long')
 200         if word:
 201             result.append(word)
 202     return unicode(delim.join(result))
 203
 204 ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 205 ### Special email test stuff begins HERE
 206 ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 207
 208 # We have two "test inboxes" here:
 209 #
 210 # EMAIL_TEST_INBOX:
 211 # ----------------
 212 #   If you're writing test views, you'll probably want to check this.
 213 #   It contains a list of MIMEText messages.
 214 #
 215 # EMAIL_TEST_MBOX_INBOX:
 216 # ----------------------
 217 #   This collects the messages from the FakeMhost inbox.  It's reslly
 218 #   just here for testing the send_email method itself.
 219 #
 220 #   Anyway this contains:
 221 #    - from
 222 #    - to: a list of email recipient addresses
 223 #    - message: not just the body, but the whole message, including
 224 #      headers, etc.
 225 #
 226 # ***IMPORTANT!***
 227 # ----------------
 228 # Before running tests that call functions which send email, you should
 229 # always call _clear_test_inboxes() to "wipe" the inboxes clean.
 230
 231 EMAIL_TEST_INBOX = []
 232 EMAIL_TEST_MBOX_INBOX = []
 233
 234
 235 class FakeMhost(object):
 236     """
 237     Just a fake mail host so we can capture and test messages
 238     from send_email
 239     """
 240     def connect(self):
 241         pass
 242
 243     def sendmail(self, from_addr, to_addrs, message):
 244         EMAIL_TEST_MBOX_INBOX.append(
 245             {'from': from_addr,
 246              'to': to_addrs,
 247              'message': message})
 248
 249 def _clear_test_inboxes():
 250     global EMAIL_TEST_INBOX
 251     global EMAIL_TEST_MBOX_INBOX
 252     EMAIL_TEST_INBOX = []
 253     EMAIL_TEST_MBOX_INBOX = []
 254
 255 ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 256 ### </Special email test stuff>
 257 ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 258
 259 def send_email(from_addr, to_addrs, subject, message_body):
 260     """
 261     Simple email sending wrapper, use this so we can capture messages
 262     for unit testing purposes.
 263
 264     Args:
 265      - from_addr: address you're sending the email from
 266      - to_addrs: list of recipient email addresses
 267      - subject: subject of the email
 268      - message_body: email body text
 269     """
 270     # TODO: make a mock mhost if testing is enabled
 271     if TESTS_ENABLED or mg_globals.email_debug_mode:
 272         mhost = FakeMhost()
 273     elif not mg_globals.email_debug_mode:
 274         mhost = smtplib.SMTP()
 275
 276     mhost.connect()
 277
 278     message = MIMEText(message_body.encode('utf-8'), 'plain', 'utf-8')
 279     message['Subject'] = subject
 280     message['From'] = from_addr
 281     message['To'] = ', '.join(to_addrs)
 282
 283     if TESTS_ENABLED:
 284         EMAIL_TEST_INBOX.append(message)
 285
 286     if getattr(mg_globals, 'email_debug_mode', False):
 287         print u"===== Email ====="
 288         print u"From address: %s" % message['From']
 289         print u"To addresses: %s" % message['To']
 290         print u"Subject: %s" % message['Subject']
 291         print u"-- Body: --"
 292         print message.get_payload(decode=True)
 293
 294     return mhost.sendmail(from_addr, to_addrs, message.as_string())
 295
 296
 297 ###################
 298 # Translation tools
 299 ###################
 300
 301
 302 TRANSLATIONS_PATH = pkg_resources.resource_filename(
 303     'mediagoblin', 'translations')
 304
 305
 306 def locale_to_lower_upper(locale):
 307     """
 308     Take a locale, regardless of style, and format it like "en-us"
 309     """
 310     if '-' in locale:
 311         lang, country = locale.split('-', 1)
 312         return '%s_%s' % (lang.lower(), country.upper())
 313     elif '_' in locale:
 314         lang, country = locale.split('_', 1)
 315         return '%s_%s' % (lang.lower(), country.upper())
 316     else:
 317         return locale.lower()
 318
 319
 320 def locale_to_lower_lower(locale):
 321     """
 322     Take a locale, regardless of style, and format it like "en_US"
 323     """
 324     if '_' in locale:
 325         lang, country = locale.split('_', 1)
 326         return '%s-%s' % (lang.lower(), country.lower())
 327     else:
 328         return locale.lower()
 329
 330
 331 def get_locale_from_request(request):
 332     """
 333     Figure out what target language is most appropriate based on the
 334     request
 335     """
 336     request_form = request.GET or request.POST
 337
 338     if request_form.has_key('lang'):
 339         return locale_to_lower_upper(request_form['lang'])
 340
 341     accept_lang_matches = request.accept_language.best_matches()
 342
 343     # Your routing can explicitly specify a target language
 344     if request.matchdict.has_key('locale'):
 345         target_lang = request.matchdict['locale']
 346     elif request.session.has_key('target_lang'):
 347         target_lang = request.session['target_lang']
 348     # Pull the first acceptable language
 349     elif accept_lang_matches:
 350         target_lang = accept_lang_matches[0]
 351     # Fall back to English
 352     else:
 353         target_lang = 'en'
 354
 355     return locale_to_lower_upper(target_lang)
 356
 357
 358 # A super strict version of the lxml.html cleaner class
 359 HTML_CLEANER = Cleaner(
 360     scripts=True,
 361     javascript=True,
 362     comments=True,
 363     style=True,
 364     links=True,
 365     page_structure=True,
 366     processing_instructions=True,
 367     embedded=True,
 368     frames=True,
 369     forms=True,
 370     annoying_tags=True,
 371     allow_tags=[
 372         'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br'],
 373     remove_unknown_tags=False, # can't be used with allow_tags
 374     safe_attrs_only=True,
 375     add_nofollow=True, # for now
 376     host_whitelist=(),
 377     whitelist_tags=set([]))
 378
 379
 380 def clean_html(html):
 381     # clean_html barfs on an empty string
 382     if not html:
 383         return u''
 384
 385     return HTML_CLEANER.clean_html(html)
 386
 387
 388 MARKDOWN_INSTANCE = markdown.Markdown(safe_mode='escape')
 389
 390
 391 def cleaned_markdown_conversion(text):
 392     """
 393     Take a block of text, run it through MarkDown, and clean its HTML.
 394     """
 395     # Markdown will do nothing with and clean_html can do nothing with
 396     # an empty string :)
 397     if not text:
 398         return u''
 399
 400     return clean_html(MARKDOWN_INSTANCE.convert(text))
 401
 402
 403 SETUP_GETTEXTS = {}
 404
 405 def setup_gettext(locale):
 406     """
 407     Setup the gettext instance based on this locale
 408     """
 409     # Later on when we have plugins we may want to enable the
 410     # multi-translations system they have so we can handle plugin
 411     # translations too
 412
 413     # TODO: fallback nicely on translations from pt_PT to pt if not
 414     # available, etc.
 415     if SETUP_GETTEXTS.has_key(locale):
 416         this_gettext = SETUP_GETTEXTS[locale]
 417     else:
 418         this_gettext = gettext.translation(
 419             'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True)
 420         if exists(locale):
 421             SETUP_GETTEXTS[locale] = this_gettext
 422
 423     mg_globals.setup_globals(
 424         translations=this_gettext)
 425
 426
 427 PAGINATION_DEFAULT_PER_PAGE = 30
 428
 429 class Pagination(object):
 430     """
 431     Pagination class for mongodb queries.
 432
 433     Initialization through __init__(self, cursor, page=1, per_page=2),
 434     get actual data slice through __call__().
 435     """
 436
 437     def __init__(self, page, cursor, per_page=PAGINATION_DEFAULT_PER_PAGE):
 438         """
 439         Initializes Pagination
 440
 441         Args:
 442          - page: requested page
 443          - per_page: number of objects per page
 444          - cursor: db cursor
 445         """
 446         self.page = page
 447         self.per_page = per_page
 448         self.cursor = cursor
 449         self.total_count = self.cursor.count()
 450
 451     def __call__(self):
 452         """
 453         Returns slice of objects for the requested page
 454         """
 455         return self.cursor.skip(
 456             (self.page - 1) * self.per_page).limit(self.per_page)
 457
 458     @property
 459     def pages(self):
 460         return int(ceil(self.total_count / float(self.per_page)))
 461
 462     @property
 463     def has_prev(self):
 464         return self.page > 1
 465
 466     @property
 467     def has_next(self):
 468         return self.page < self.pages
 469
 470     def iter_pages(self, left_edge=2, left_current=2,
 471                    right_current=5, right_edge=2):
 472         last = 0
 473         for num in xrange(1, self.pages + 1):
 474             if num <= left_edge or \
 475                (num > self.page - left_current - 1 and \
 476                 num < self.page + right_current) or \
 477                num > self.pages - right_edge:
 478                 if last + 1 != num:
 479                     yield None
 480                 yield num
 481                 last = num
 482
 483     def get_page_url_explicit(self, base_url, get_params, page_no):
 484         """
 485         Get a page url by adding a page= parameter to the base url
 486         """
 487         new_get_params = copy.copy(get_params or {})
 488         new_get_params['page'] = page_no
 489         return "%s?%s" % (
 490             base_url, urllib.urlencode(new_get_params))
 491
 492     def get_page_url(self, request, page_no):
 493         """
 494         Get a new page url based of the request, and the new page number.
 495
 496         This is a nice wrapper around get_page_url_explicit()
 497         """
 498         return self.get_page_url_explicit(
 499             request.path_info, request.GET, page_no)