Commit | Line | Data |
---|---|---|
e231d9e8 CAW |
1 | # GNU MediaGoblin -- federated, autonomous media hosting |
2 | # Copyright (C) 2011 Free Software Foundation, Inc | |
3 | # | |
4 | # This program is free software: you can redistribute it and/or modify | |
5 | # it under the terms of the GNU Affero General Public License as published by | |
6 | # the Free Software Foundation, either version 3 of the License, or | |
7 | # (at your option) any later version. | |
8 | # | |
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU Affero General Public License for more details. | |
13 | # | |
14 | # You should have received a copy of the GNU Affero General Public License | |
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | ||
17 | import os | |
694c2351 | 18 | import sys |
e231d9e8 CAW |
19 | |
20 | from paste.deploy.converters import asbool, asint, aslist | |
21 | ||
22 | ||
23 | KNOWN_CONFIG_BOOLS = [ | |
24 | 'CELERY_RESULT_PERSISTENT', | |
25 | 'CELERY_CREATE_MISSING_QUEUES', | |
26 | 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY', | |
27 | 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS', | |
28 | 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED', | |
29 | 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE', | |
30 | 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED', | |
31 | 'CELERY_SEND_TASK_ERROR_EMAILS', | |
32 | 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT', | |
33 | 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS', | |
34 | ] | |
35 | ||
36 | KNOWN_CONFIG_INTS = [ | |
37 | 'CELERYD_CONCURRENCY', | |
38 | 'CELERYD_PREFETCH_MULTIPLIER', | |
39 | 'CELERY_AMQP_TASK_RESULT_EXPIRES', | |
40 | 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX', | |
41 | 'REDIS_PORT', 'REDIS_DB', | |
42 | 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT', | |
43 | 'CELERY_BROKER_CONNECTION_MAX_RETRIES', | |
44 | 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS', | |
45 | 'CELERY_DEFAULT_RATE_LIMIT', # ?? | |
46 | 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT', | |
47 | 'CELERYD_TASK_SOFT_TIME_LIMIT', | |
48 | 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL', | |
49 | ] | |
50 | ||
51 | KNOWN_CONFIG_FLOATS = [ | |
52 | 'CELERYD_ETA_SCHEDULER_PRECISION', | |
53 | ] | |
54 | ||
55 | KNOWN_CONFIG_LISTS = [ | |
56 | 'CELERY_ROUTES', 'CELERY_IMPORTS', | |
57 | ] | |
58 | ||
59 | ||
60 | ## Needs special processing: | |
61 | # ADMINS, ??? | |
62 | # there are a lot more; we should list here or process specially. | |
63 | ||
64 | ||
65 | def asfloat(obj): | |
66 | try: | |
67 | return float(obj) | |
68 | except (TypeError, ValueError), e: | |
69 | raise ValueError( | |
70 | "Bad float value: %r" % obj) | |
71 | ||
72 | ||
88816492 CAW |
73 | MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] |
74 | ||
694c2351 CAW |
75 | DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module' |
76 | ||
77 | def setup_celery_from_config(app_config, global_config, | |
78 | settings_module=DEFAULT_SETTINGS_MODULE, | |
79 | set_environ=True): | |
e231d9e8 CAW |
80 | """ |
81 | Take a mediagoblin app config and the global config from a paste | |
82 | factory and try to set up a celery settings module from this. | |
694c2351 CAW |
83 | |
84 | Args: | |
85 | - app_config: the application config section | |
86 | - global_config: the entire paste config, all sections | |
87 | - settings_module: the module to populate, as a string | |
88 | - set_environ: if set, this will CELERY_CONFIG_MODULE to the | |
89 | settings_module | |
e231d9e8 CAW |
90 | """ |
91 | if asbool(app_config.get('use_celery_environment_var')) == True: | |
92 | # Don't setup celery based on our config file. | |
93 | return | |
94 | ||
95 | celery_conf_section = app_config.get('celery_section', 'celery') | |
96 | if global_config.has_key(celery_conf_section): | |
97 | celery_conf = global_config[celery_conf_section] | |
98 | else: | |
99 | celery_conf = {} | |
100 | ||
101 | celery_settings = {} | |
102 | ||
103 | # set up mongodb stuff | |
1c61a6ca CAW |
104 | celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' |
105 | if not celery_settings.has_key('BROKER_BACKEND'): | |
106 | celery_settings['BROKER_BACKEND'] = 'mongodb' | |
107 | ||
e231d9e8 | 108 | celery_mongo_settings = {} |
1c61a6ca | 109 | |
e231d9e8 CAW |
110 | if app_config.has_key('db_host'): |
111 | celery_mongo_settings['host'] = app_config['db_host'] | |
1c61a6ca CAW |
112 | if celery_settings['BROKER_BACKEND'] == 'mongodb': |
113 | celery_settings['BROKER_HOST'] = app_config['db_host'] | |
e231d9e8 CAW |
114 | if app_config.has_key('db_port'): |
115 | celery_mongo_settings['port'] = asint(app_config['db_port']) | |
1c61a6ca CAW |
116 | if celery_settings['BROKER_BACKEND'] == 'mongodb': |
117 | celery_settings['BROKER_PORT'] = asint(app_config['db_port']) | |
e231d9e8 CAW |
118 | celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') |
119 | ||
120 | celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings | |
e231d9e8 CAW |
121 | |
122 | # Add anything else | |
123 | for key, value in celery_conf.iteritems(): | |
124 | key = key.upper() | |
125 | if key in KNOWN_CONFIG_BOOLS: | |
126 | value = asbool(value) | |
1c61a6ca | 127 | elif key in KNOWN_CONFIG_INTS: |
e231d9e8 | 128 | value = asint(value) |
1c61a6ca | 129 | elif key in KNOWN_CONFIG_FLOATS: |
e231d9e8 | 130 | value = asfloat(value) |
1c61a6ca | 131 | elif key in KNOWN_CONFIG_LISTS: |
e231d9e8 CAW |
132 | value = aslist(value) |
133 | celery_settings[key] = value | |
134 | ||
88816492 CAW |
135 | # add mandatory celery imports |
136 | celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) | |
137 | celery_imports.extend(MANDATORY_CELERY_IMPORTS) | |
138 | ||
694c2351 CAW |
139 | __import__(settings_module) |
140 | this_module = sys.modules[settings_module] | |
e231d9e8 | 141 | |
ef30978a | 142 | for key, value in celery_settings.iteritems(): |
694c2351 CAW |
143 | setattr(this_module, key, value) |
144 | ||
145 | if set_environ: | |
146 | os.environ['CELERY_CONFIG_MODULE'] = settings_module |