From: Christopher Allan Webber Date: Mon, 26 Mar 2012 16:14:11 +0000 (-0500) Subject: SQL based tests and refactored Celery setup stuff X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=d693f6bd867871ba084f0da0ff8ba5a667174fc7;p=mediagoblin.git SQL based tests and refactored Celery setup stuff - Changed config files of test configs to use SQL - Updated celery initialization tools, factored them to be able to use the "big instance" application stuff --- diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 27698170..5415b997 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -65,14 +65,13 @@ def gather_database_data(media_types): return managed_dbdata -def dbupdate(args): +def run_dbupdate(app_config): """ Initialize or migrate the database as specified by the config file. Will also initialize or migrate all extensions (media types, and in the future, plugins) """ - globa_config, app_config = setup_global_and_app_config(args.conf_file) # Gather information from all media managers / projects dbdatas = gather_database_data(app_config['media_types']) @@ -87,3 +86,8 @@ def dbupdate(args): for dbdata in dbdatas: migration_manager = dbdata.make_migration_manager(Session()) migration_manager.init_or_migrate() + + +def dbupdate(args): + global_config, app_config = setup_global_and_app_config(args.conf_file) + run_dbupdate(app_config) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 67b020c3..fc595ea7 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -17,28 +17,18 @@ import os import sys +from celery import Celery + MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task'] DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' -def setup_celery_from_config(app_config, global_config, - settings_module=DEFAULT_SETTINGS_MODULE, - force_celery_always_eager=False, - set_environ=True): +def get_celery_settings_dict(app_config, global_config, + force_celery_always_eager=False): """ - Take a mediagoblin app config and try to set up a celery settings - module from this. - - Args: - - app_config: the application config section - - global_config: the entire ConfigObj loaded config, all sections - - settings_module: the module to populate, as a string - - force_celery_always_eager: whether or not to force celery into - always eager mode; good for development and small installs - - set_environ: if set, this will CELERY_CONFIG_MODULE to the - settings_module + Get a celery settings dictionary from reading the config """ if 'celery' in global_config: celery_conf = global_config['celery'] @@ -61,6 +51,41 @@ def setup_celery_from_config(app_config, global_config, celery_settings['CELERY_ALWAYS_EAGER'] = True celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + return celery_settings + + +def setup_celery_app(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False): + """ + Setup celery without using terrible setup-celery-module hacks. + """ + celery_settings = get_celery_settings_dict( + app_config, global_config, force_celery_always_eager) + celery_app = Celery() + celery_app.config_from_object(celery_settings) + + +def setup_celery_from_config(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False, + set_environ=True): + """ + Take a mediagoblin app config and try to set up a celery settings + module from this. + + Args: + - app_config: the application config section + - global_config: the entire ConfigObj loaded config, all sections + - settings_module: the module to populate, as a string + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs + - set_environ: if set, this will CELERY_CONFIG_MODULE to the + settings_module + """ + celery_settings = get_celery_settings_dict( + app_config, global_config, force_celery_always_eager) + __import__(settings_module) this_module = sys.modules[settings_module] diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index 01bf0972..fd610ecc 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -2,7 +2,9 @@ direct_remote_path = /test_static/ email_sender_address = "notice@mediagoblin.example.org" email_debug_mode = true -db_name = __mediagoblin_tests__ + +# Use an in-memory database +sql_engine = "sqlite:///%(here)s/test_user_dev/mediagoblin.db" # tag parsing tags_max_length = 50 @@ -27,3 +29,5 @@ lock_dir = %(here)s/test_user_dev/beaker/cache/lock [celery] CELERY_ALWAYS_EAGER = true +CELERY_RESULT_DBURI = "sqlite:///%(here)s/test_user_dev/celery.db" +BROKER_HOST = "sqlite:///%(here)s/test_user_dev/kombu.db" diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 7cf355b0..a99173e8 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -26,8 +26,11 @@ from mediagoblin.tools import testing from mediagoblin.init.config import read_mediagoblin_config from mediagoblin.decorators import _make_safe from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.db.sql.base import Session from mediagoblin.meddleware import BaseMeddleware from mediagoblin.auth.lib import bcrypt_gen_password_hash +from mediagoblin.gmg_commands.dbupdate import run_dbupdate +from mediagoblin.init.celery import setup_celery_app MEDIAGOBLIN_TEST_DB_NAME = u'__mediagoblin_tests__' @@ -125,26 +128,19 @@ def get_test_app(dump_old_app=True): global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG) app_config = global_config['mediagoblin'] - # Wipe database - # @@: For now we're dropping collections, but we could also just - # collection.remove() ? - connection, db = setup_connection_and_db_from_config(app_config) - assert db.name == MEDIAGOBLIN_TEST_DB_NAME - - collections_to_wipe = [ - collection - for collection in db.collection_names() - if not collection.startswith('system.')] - - for collection in collections_to_wipe: - db.drop_collection(collection) - - # TODO: Drop and recreate indexes + # Run database setup/migrations + run_dbupdate(app_config) # setup app and return test_app = loadapp( 'config:' + TEST_SERVER_CONFIG) + Session.rollback() + Session.remove() + + # Re-setup celery + setup_celery_app(app_config, global_config) + # Insert the TestingMeddleware, which can do some # sanity checks on every request/response. # Doing it this way is probably not the cleanest way.