mediagoblin/celery_setup/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011 Free Software Foundation, Inc
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19
  20 from paste.deploy.converters import asbool, asint, aslist
  21
  22
  23 KNOWN_CONFIG_BOOLS = [
  24     'CELERY_RESULT_PERSISTENT',
  25     'CELERY_CREATE_MISSING_QUEUES',
  26     'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY',
  27     'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS',
  28     'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED',
  29     'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE',
  30     'CELERY_STORE_ERRORS_EVEN_IF_IGNORED',
  31     'CELERY_SEND_TASK_ERROR_EMAILS',
  32     'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT',
  33     'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS',
  34     ]
  35
  36 KNOWN_CONFIG_INTS = [
  37     'CELERYD_CONCURRENCY',
  38     'CELERYD_PREFETCH_MULTIPLIER',
  39     'CELERY_AMQP_TASK_RESULT_EXPIRES',
  40     'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX',
  41     'REDIS_PORT', 'REDIS_DB',
  42     'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT',
  43     'CELERY_BROKER_CONNECTION_MAX_RETRIES',
  44     'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS',
  45     'CELERY_DEFAULT_RATE_LIMIT', # ??
  46     'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT',
  47     'CELERYD_TASK_SOFT_TIME_LIMIT',
  48     'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL',
  49     ]
  50
  51 KNOWN_CONFIG_FLOATS = [
  52     'CELERYD_ETA_SCHEDULER_PRECISION',
  53     ]
  54
  55 KNOWN_CONFIG_LISTS = [
  56     'CELERY_ROUTES', 'CELERY_IMPORTS',
  57     ]
  58
  59
  60 ## Needs special processing:
  61 # ADMINS, ???
  62 # there are a lot more; we should list here or process specially.
  63
  64
  65 def asfloat(obj):
  66     try:
  67         return float(obj)
  68     except (TypeError, ValueError), e:
  69         raise ValueError(
  70             "Bad float value: %r" % obj)
  71
  72
  73 MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media']
  74
  75 DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module'
  76
  77 def setup_celery_from_config(app_config, global_config,
  78                              settings_module=DEFAULT_SETTINGS_MODULE,
  79                              force_celery_always_eager=False,
  80                              set_environ=True):
  81     """
  82     Take a mediagoblin app config and the global config from a paste
  83     factory and try to set up a celery settings module from this.
  84
  85     Args:
  86     - app_config: the application config section
  87     - global_config: the entire paste config, all sections
  88     - settings_module: the module to populate, as a string
  89     -
  90     - set_environ: if set, this will CELERY_CONFIG_MODULE to the
  91       settings_module
  92     """
  93     if asbool(app_config.get('use_celery_environment_var')) == True:
  94         # Don't setup celery based on our config file.
  95         return
  96
  97     celery_conf_section = app_config.get('celery_section', 'celery')
  98     if global_config.has_key(celery_conf_section):
  99         celery_conf = global_config[celery_conf_section]
 100     else:
 101         celery_conf = {}
 102
 103     celery_settings = {}
 104
 105     # set up mongodb stuff
 106     celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
 107     if not celery_settings.has_key('BROKER_BACKEND'):
 108         celery_settings['BROKER_BACKEND'] = 'mongodb'
 109
 110     celery_mongo_settings = {}
 111
 112     if app_config.has_key('db_host'):
 113         celery_mongo_settings['host'] = app_config['db_host']
 114         if celery_settings['BROKER_BACKEND'] == 'mongodb':
 115             celery_settings['BROKER_HOST'] = app_config['db_host']
 116     if app_config.has_key('db_port'):
 117         celery_mongo_settings['port'] = asint(app_config['db_port'])
 118         if celery_settings['BROKER_BACKEND'] == 'mongodb':
 119             celery_settings['BROKER_PORT'] = asint(app_config['db_port'])
 120     celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
 121
 122     celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
 123
 124     # Add anything else
 125     for key, value in celery_conf.iteritems():
 126         key = key.upper()
 127         if key in KNOWN_CONFIG_BOOLS:
 128             value = asbool(value)
 129         elif key in KNOWN_CONFIG_INTS:
 130             value = asint(value)
 131         elif key in KNOWN_CONFIG_FLOATS:
 132             value = asfloat(value)
 133         elif key in KNOWN_CONFIG_LISTS:
 134             value = aslist(value)
 135         celery_settings[key] = value
 136
 137     # add mandatory celery imports
 138     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
 139     celery_imports.extend(MANDATORY_CELERY_IMPORTS)
 140
 141     if force_celery_always_eager:
 142         celery_settings['CELERY_ALWAYS_EAGER'] = True
 143         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
 144
 145     __import__(settings_module)
 146     this_module = sys.modules[settings_module]
 147
 148     for key, value in celery_settings.iteritems():
 149         setattr(this_module, key, value)
 150
 151     if set_environ:
 152         os.environ['CELERY_CONFIG_MODULE'] = settings_module