mediagoblin/init/celery/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19
  20 from celery import Celery
  21 from mediagoblin.tools.pluginapi import callable_runall
  22
  23
  24 MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task']
  25
  26 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
  27
  28
  29 def get_celery_settings_dict(app_config, global_config,
  30                              force_celery_always_eager=False):
  31     """
  32     Get a celery settings dictionary from reading the config
  33     """
  34     if 'celery' in global_config:
  35         celery_conf = global_config['celery']
  36     else:
  37         celery_conf = {}
  38
  39     celery_settings = {}
  40
  41     # Add all celery settings from config
  42     for key, value in celery_conf.iteritems():
  43         celery_settings[key] = value
  44
  45     # TODO: use default result stuff here if it exists
  46
  47     # add mandatory celery imports
  48     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
  49     celery_imports.extend(MANDATORY_CELERY_IMPORTS)
  50
  51     if force_celery_always_eager:
  52         celery_settings['CELERY_ALWAYS_EAGER'] = True
  53         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
  54
  55     return celery_settings
  56
  57
  58 def setup_celery_app(app_config, global_config,
  59                      settings_module=DEFAULT_SETTINGS_MODULE,
  60                      force_celery_always_eager=False):
  61     """
  62     Setup celery without using terrible setup-celery-module hacks.
  63     """
  64     celery_settings = get_celery_settings_dict(
  65         app_config, global_config, force_celery_always_eager)
  66     celery_app = Celery()
  67     celery_app.config_from_object(celery_settings)
  68
  69     callable_runall('celery_setup', celery_app)
  70
  71
  72 def setup_celery_from_config(app_config, global_config,
  73                              settings_module=DEFAULT_SETTINGS_MODULE,
  74                              force_celery_always_eager=False,
  75                              set_environ=True):
  76     """
  77     Take a mediagoblin app config and try to set up a celery settings
  78     module from this.
  79
  80     Args:
  81     - app_config: the application config section
  82     - global_config: the entire ConfigObj loaded config, all sections
  83     - settings_module: the module to populate, as a string
  84     - force_celery_always_eager: whether or not to force celery into
  85       always eager mode; good for development and small installs
  86     - set_environ: if set, this will CELERY_CONFIG_MODULE to the
  87       settings_module
  88     """
  89     celery_settings = get_celery_settings_dict(
  90         app_config, global_config, force_celery_always_eager)
  91
  92     __import__(settings_module)
  93     this_module = sys.modules[settings_module]
  94
  95     for key, value in celery_settings.iteritems():
  96         setattr(this_module, key, value)
  97
  98     if set_environ:
  99         os.environ['CELERY_CONFIG_MODULE'] = settings_module