| 1 | # GNU MediaGoblin -- federated, autonomous media hosting |
| 2 | # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. |
| 3 | # |
| 4 | # This program is free software: you can redistribute it and/or modify |
| 5 | # it under the terms of the GNU Affero General Public License as published by |
| 6 | # the Free Software Foundation, either version 3 of the License, or |
| 7 | # (at your option) any later version. |
| 8 | # |
| 9 | # This program is distributed in the hope that it will be useful, |
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | # GNU Affero General Public License for more details. |
| 13 | # |
| 14 | # You should have received a copy of the GNU Affero General Public License |
| 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 16 | |
| 17 | import logging |
| 18 | |
| 19 | import six |
| 20 | from alembic import command |
| 21 | from sqlalchemy.orm import sessionmaker |
| 22 | |
| 23 | from mediagoblin.db.open import setup_connection_and_db_from_config |
| 24 | from mediagoblin.db.migration_tools import ( |
| 25 | MigrationManager, build_alembic_config, populate_table_foundations) |
| 26 | from mediagoblin.init import setup_global_and_app_config |
| 27 | from mediagoblin.tools.common import import_component |
| 28 | |
| 29 | _log = logging.getLogger(__name__) |
| 30 | logging.basicConfig() |
| 31 | ## Let's not set the level as debug by default to avoid confusing users :) |
| 32 | # _log.setLevel(logging.DEBUG) |
| 33 | |
| 34 | |
| 35 | def dbupdate_parse_setup(subparser): |
| 36 | pass |
| 37 | |
| 38 | |
| 39 | class DatabaseData(object): |
| 40 | def __init__(self, name, models, migrations): |
| 41 | self.name = name |
| 42 | self.models = models |
| 43 | self.migrations = migrations |
| 44 | |
| 45 | def make_migration_manager(self, session): |
| 46 | return MigrationManager( |
| 47 | self.name, self.models, self.migrations, session) |
| 48 | |
| 49 | |
| 50 | def gather_database_data(plugins): |
| 51 | """ |
| 52 | Gather all database data relevant to the extensions installed. |
| 53 | |
| 54 | Gather all database data relevant to the extensions we have |
| 55 | installed so we can do migrations and table initialization. |
| 56 | |
| 57 | Returns a list of DatabaseData objects. |
| 58 | """ |
| 59 | managed_dbdata = [] |
| 60 | |
| 61 | # Add main first |
| 62 | from mediagoblin.db.models import MODELS as MAIN_MODELS |
| 63 | from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS |
| 64 | |
| 65 | managed_dbdata.append( |
| 66 | DatabaseData( |
| 67 | u'__main__', MAIN_MODELS, MAIN_MIGRATIONS)) |
| 68 | |
| 69 | for plugin in plugins: |
| 70 | try: |
| 71 | models = import_component('{0}.models:MODELS'.format(plugin)) |
| 72 | except ImportError as exc: |
| 73 | _log.debug('No models found for {0}: {1}'.format( |
| 74 | plugin, |
| 75 | exc)) |
| 76 | |
| 77 | models = [] |
| 78 | except AttributeError as exc: |
| 79 | _log.warning('Could not find MODELS in {0}.models, have you ' |
| 80 | 'forgotten to add it? ({1})'.format(plugin, exc)) |
| 81 | models = [] |
| 82 | |
| 83 | try: |
| 84 | migrations = import_component('{0}.migrations:MIGRATIONS'.format( |
| 85 | plugin)) |
| 86 | except ImportError as exc: |
| 87 | _log.debug('No migrations found for {0}: {1}'.format( |
| 88 | plugin, |
| 89 | exc)) |
| 90 | |
| 91 | migrations = {} |
| 92 | except AttributeError as exc: |
| 93 | _log.debug('Could not find MIGRATIONS in {0}.migrations, have you' |
| 94 | 'forgotten to add it? ({1})'.format(plugin, exc)) |
| 95 | migrations = {} |
| 96 | |
| 97 | if models: |
| 98 | managed_dbdata.append( |
| 99 | DatabaseData(plugin, models, migrations)) |
| 100 | |
| 101 | return managed_dbdata |
| 102 | |
| 103 | |
| 104 | def run_foundations(db, global_config): |
| 105 | """ |
| 106 | Gather foundations data and run it. |
| 107 | """ |
| 108 | from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS |
| 109 | all_foundations = [(u"__main__", MAIN_FOUNDATIONS)] |
| 110 | |
| 111 | Session = sessionmaker(bind=db.engine) |
| 112 | session = Session() |
| 113 | |
| 114 | plugins = global_config.get('plugins', {}) |
| 115 | |
| 116 | for plugin in plugins: |
| 117 | try: |
| 118 | foundations = import_component( |
| 119 | '{0}.models:FOUNDATIONS'.format(plugin)) |
| 120 | all_foundations.append((plugin, foundations)) |
| 121 | except ImportError as exc: |
| 122 | continue |
| 123 | except AttributeError as exc: |
| 124 | continue |
| 125 | |
| 126 | for name, foundations in all_foundations: |
| 127 | populate_table_foundations(session, foundations, name) |
| 128 | |
| 129 | |
| 130 | def run_alembic_migrations(db, app_config, global_config): |
| 131 | """Initialize a database and runs all Alembic migrations.""" |
| 132 | Session = sessionmaker(bind=db.engine) |
| 133 | session = Session() |
| 134 | cfg = build_alembic_config(global_config, None, session) |
| 135 | |
| 136 | res = command.upgrade(cfg, 'heads') |
| 137 | session.commit() |
| 138 | return res |
| 139 | |
| 140 | |
| 141 | def run_dbupdate(app_config, global_config): |
| 142 | """ |
| 143 | Initialize or migrate the database as specified by the config file. |
| 144 | |
| 145 | Will also initialize or migrate all extensions (media types, and |
| 146 | in the future, plugins) |
| 147 | """ |
| 148 | # Set up the database |
| 149 | db = setup_connection_and_db_from_config(app_config, migrations=True) |
| 150 | |
| 151 | # Do we have migrations |
| 152 | should_run_sqam_migrations = db.engine.has_table("core__migrations") and \ |
| 153 | sqam_migrations_to_run(db, app_config, |
| 154 | global_config) |
| 155 | |
| 156 | # Looks like a fresh database! |
| 157 | # (We set up this variable here because doing "run_all_migrations" below |
| 158 | # will change things.) |
| 159 | fresh_database = ( |
| 160 | not db.engine.has_table("core__migrations") and |
| 161 | not db.engine.has_table("alembic_version")) |
| 162 | |
| 163 | # Run the migrations |
| 164 | if should_run_sqam_migrations: |
| 165 | run_all_migrations(db, app_config, global_config) |
| 166 | |
| 167 | run_alembic_migrations(db, app_config, global_config) |
| 168 | |
| 169 | # If this was our first time initializing the database, |
| 170 | # we must lay down the foundations |
| 171 | if fresh_database: |
| 172 | run_foundations(db, global_config) |
| 173 | |
| 174 | |
| 175 | def run_all_migrations(db, app_config, global_config): |
| 176 | """Initialize or migrates a database. |
| 177 | |
| 178 | Initializes or migrates a database that already has a |
| 179 | connection setup and also initializes or migrates all |
| 180 | extensions based on the config files. |
| 181 | |
| 182 | It can be used to initialize an in-memory database for |
| 183 | testing. |
| 184 | """ |
| 185 | # Gather information from all media managers / projects |
| 186 | dbdatas = gather_database_data( |
| 187 | list(global_config.get('plugins', {}).keys())) |
| 188 | |
| 189 | Session = sessionmaker(bind=db.engine) |
| 190 | |
| 191 | # Setup media managers for all dbdata, run init/migrate and print info |
| 192 | # For each component, create/migrate tables |
| 193 | for dbdata in dbdatas: |
| 194 | migration_manager = dbdata.make_migration_manager(Session()) |
| 195 | migration_manager.init_or_migrate() |
| 196 | |
| 197 | |
| 198 | def sqam_migrations_to_run(db, app_config, global_config): |
| 199 | """ |
| 200 | Check whether any plugins have sqlalchemy-migrate migrations left to run. |
| 201 | |
| 202 | This is a kludge so we can transition away from sqlalchemy-migrate |
| 203 | except where necessary. |
| 204 | """ |
| 205 | # @@: This shares a lot of code with run_all_migrations, but both |
| 206 | # are legacy and will be removed at some point. |
| 207 | |
| 208 | # Gather information from all media managers / projects |
| 209 | dbdatas = gather_database_data( |
| 210 | list(global_config.get('plugins', {}).keys())) |
| 211 | |
| 212 | Session = sessionmaker(bind=db.engine) |
| 213 | |
| 214 | # We can bail out early if it turns out that sqlalchemy-migrate |
| 215 | # was never installed with any migrations |
| 216 | from mediagoblin.db.models import MigrationData |
| 217 | if Session().query(MigrationData).filter_by( |
| 218 | name=u"__main__").first() is None: |
| 219 | return False |
| 220 | |
| 221 | # Setup media managers for all dbdata, run init/migrate and print info |
| 222 | # For each component, create/migrate tables |
| 223 | for dbdata in dbdatas: |
| 224 | migration_manager = dbdata.make_migration_manager(Session()) |
| 225 | if migration_manager.migrations_to_run(): |
| 226 | # If *any* migration managers have migrations to run, |
| 227 | # we'll have to run them. |
| 228 | return True |
| 229 | |
| 230 | # Otherwise, scot free! |
| 231 | return False |
| 232 | |
| 233 | |
| 234 | def dbupdate(args): |
| 235 | global_config, app_config = setup_global_and_app_config(args.conf_file) |
| 236 | run_dbupdate(app_config, global_config) |