mediagoblin/celery_setup/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011 Free Software Foundation, Inc
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19
  20
  21 MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media']
  22
  23 DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module'
  24
  25
  26 def setup_celery_from_config(app_config, global_config,
  27                              settings_module=DEFAULT_SETTINGS_MODULE,
  28                              force_celery_always_eager=False,
  29                              set_environ=True):
  30     """
  31     Take a mediagoblin app config and try to set up a celery settings
  32     module from this.
  33
  34     Args:
  35     - app_config: the application config section
  36     - global_config: the entire ConfigObj loaded config, all sections
  37     - settings_module: the module to populate, as a string
  38     - force_celery_always_eager: whether or not to force celery into
  39       always eager mode; good for development and small installs
  40     - set_environ: if set, this will CELERY_CONFIG_MODULE to the
  41       settings_module
  42     """
  43     if app_config.get('celery_setup_elsewhere') == True:
  44         # Don't setup celery based on our config file.
  45         return
  46
  47     if global_config.has_key('celery'):
  48         celery_conf = global_config['celery']
  49     else:
  50         celery_conf = {}
  51
  52     celery_settings = {}
  53
  54     # set up mongodb stuff
  55     celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
  56     if not celery_settings.has_key('BROKER_BACKEND'):
  57         celery_settings['BROKER_BACKEND'] = 'mongodb'
  58
  59     celery_mongo_settings = {}
  60
  61     if app_config.has_key('db_host'):
  62         celery_mongo_settings['host'] = app_config['db_host']
  63         if celery_settings['BROKER_BACKEND'] == 'mongodb':
  64             celery_settings['BROKER_HOST'] = app_config['db_host']
  65     if app_config.has_key('db_port'):
  66         celery_mongo_settings['port'] = app_config['db_port']
  67         if celery_settings['BROKER_BACKEND'] == 'mongodb':
  68             celery_settings['BROKER_PORT'] = app_config['db_port']
  69     celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
  70
  71     celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
  72
  73     # Add anything else
  74     for key, value in celery_conf.iteritems():
  75         key = key.upper()
  76         celery_settings[key] = value
  77
  78     # add mandatory celery imports
  79     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
  80     celery_imports.extend(MANDATORY_CELERY_IMPORTS)
  81
  82     if force_celery_always_eager:
  83         celery_settings['CELERY_ALWAYS_EAGER'] = True
  84         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
  85
  86     __import__(settings_module)
  87     this_module = sys.modules[settings_module]
  88
  89     for key, value in celery_settings.iteritems():
  90         setattr(this_module, key, value)
  91
  92     if set_environ:
  93         os.environ['CELERY_CONFIG_MODULE'] = settings_module