mediagoblin/init/celery/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19
  20
  21 MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing']
  22
  23 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
  24
  25
  26 def setup_celery_from_config(app_config, global_config,
  27                              settings_module=DEFAULT_SETTINGS_MODULE,
  28                              force_celery_always_eager=False,
  29                              set_environ=True):
  30     """
  31     Take a mediagoblin app config and try to set up a celery settings
  32     module from this.
  33
  34     Args:
  35     - app_config: the application config section
  36     - global_config: the entire ConfigObj loaded config, all sections
  37     - settings_module: the module to populate, as a string
  38     - force_celery_always_eager: whether or not to force celery into
  39       always eager mode; good for development and small installs
  40     - set_environ: if set, this will CELERY_CONFIG_MODULE to the
  41       settings_module
  42     """
  43     if 'celery' in global_config:
  44         celery_conf = global_config['celery']
  45     else:
  46         celery_conf = {}
  47
  48     celery_settings = {}
  49
  50     # set up mongodb stuff
  51     celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
  52     if 'BROKER_BACKEND' not in celery_settings:
  53         celery_settings['BROKER_BACKEND'] = 'mongodb'
  54
  55     celery_mongo_settings = {}
  56
  57     if 'db_host' in app_config:
  58         celery_mongo_settings['host'] = app_config['db_host']
  59         if celery_settings['BROKER_BACKEND'] == 'mongodb':
  60             celery_settings['BROKER_HOST'] = app_config['db_host']
  61     if 'db_port' in app_config:
  62         celery_mongo_settings['port'] = app_config['db_port']
  63         if celery_settings['BROKER_BACKEND'] == 'mongodb':
  64             celery_settings['BROKER_PORT'] = app_config['db_port']
  65     celery_mongo_settings['database'] = app_config['db_name']
  66
  67     celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
  68
  69     # Add anything else
  70     for key, value in celery_conf.iteritems():
  71         key = key.upper()
  72         celery_settings[key] = value
  73
  74     # add mandatory celery imports
  75     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
  76     celery_imports.extend(MANDATORY_CELERY_IMPORTS)
  77
  78     if force_celery_always_eager:
  79         celery_settings['CELERY_ALWAYS_EAGER'] = True
  80         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
  81
  82     __import__(settings_module)
  83     this_module = sys.modules[settings_module]
  84
  85     for key, value in celery_settings.iteritems():
  86         setattr(this_module, key, value)
  87
  88     if set_environ:
  89         os.environ['CELERY_CONFIG_MODULE'] = settings_module