mediagoblin/gmg_commands/dbupdate.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import logging
  18
  19 import six
  20 from alembic import command
  21 from sqlalchemy.orm import sessionmaker
  22
  23 from mediagoblin.db.open import setup_connection_and_db_from_config
  24 from mediagoblin.db.migration_tools import (
  25     MigrationManager, build_alembic_config, populate_table_foundations)
  26 from mediagoblin.init import setup_global_and_app_config
  27 from mediagoblin.tools.common import import_component
  28
  29 _log = logging.getLogger(__name__)
  30 logging.basicConfig()
  31 ## Let's not set the level as debug by default to avoid confusing users :)
  32 # _log.setLevel(logging.DEBUG)
  33
  34
  35 def dbupdate_parse_setup(subparser):
  36     pass
  37
  38
  39 class DatabaseData(object):
  40     def __init__(self, name, models, migrations):
  41         self.name = name
  42         self.models = models
  43         self.migrations = migrations
  44
  45     def make_migration_manager(self, session):
  46         return MigrationManager(
  47             self.name, self.models, self.migrations, session)
  48
  49
  50 def gather_database_data(plugins):
  51     """
  52     Gather all database data relevant to the extensions installed.
  53
  54     Gather all database data relevant to the extensions we have
  55     installed so we can do migrations and table initialization.
  56
  57     Returns a list of DatabaseData objects.
  58     """
  59     managed_dbdata = []
  60
  61     # Add main first
  62     from mediagoblin.db.models import MODELS as MAIN_MODELS
  63     from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS
  64
  65     managed_dbdata.append(
  66         DatabaseData(
  67             u'__main__', MAIN_MODELS, MAIN_MIGRATIONS))
  68
  69     for plugin in plugins:
  70         try:
  71             models = import_component('{0}.models:MODELS'.format(plugin))
  72         except ImportError as exc:
  73             _log.debug('No models found for {0}: {1}'.format(
  74                 plugin,
  75                 exc))
  76
  77             models = []
  78         except AttributeError as exc:
  79             _log.warning('Could not find MODELS in {0}.models, have you '
  80                          'forgotten to add it? ({1})'.format(plugin, exc))
  81             models = []
  82
  83         try:
  84             migrations = import_component('{0}.migrations:MIGRATIONS'.format(
  85                 plugin))
  86         except ImportError as exc:
  87             _log.debug('No migrations found for {0}: {1}'.format(
  88                 plugin,
  89                 exc))
  90
  91             migrations = {}
  92         except AttributeError as exc:
  93             _log.debug('Could not find MIGRATIONS in {0}.migrations, have you'
  94                        'forgotten to add it? ({1})'.format(plugin, exc))
  95             migrations = {}
  96
  97         if models:
  98             managed_dbdata.append(
  99                 DatabaseData(plugin, models, migrations))
 100
 101     return managed_dbdata
 102
 103
 104 def run_foundations(db, global_config):
 105     """
 106     Gather foundations data and run it.
 107     """
 108     from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
 109     all_foundations = [(u"__main__", MAIN_FOUNDATIONS)]
 110
 111     Session = sessionmaker(bind=db.engine)
 112     session = Session()
 113
 114     plugins = global_config.get('plugins', {})
 115
 116     for plugin in plugins:
 117         try:
 118             foundations = import_component(
 119                 '{0}.models:FOUNDATIONS'.format(plugin))
 120             all_foundations.append((plugin, foundations))
 121         except ImportError as exc:
 122             continue
 123         except AttributeError as exc:
 124             continue
 125
 126     for name, foundations in all_foundations:
 127         populate_table_foundations(session, foundations, name)
 128
 129
 130 def run_alembic_migrations(db, app_config, global_config):
 131     """Initialize a database and runs all Alembic migrations."""
 132     Session = sessionmaker(bind=db.engine)
 133     session = Session()
 134     cfg = build_alembic_config(global_config, None, session)
 135
 136     return command.upgrade(cfg, 'heads')
 137
 138
 139 def run_dbupdate(app_config, global_config):
 140     """
 141     Initialize or migrate the database as specified by the config file.
 142
 143     Will also initialize or migrate all extensions (media types, and
 144     in the future, plugins)
 145     """
 146     # Set up the database
 147     db = setup_connection_and_db_from_config(app_config, migrations=True)
 148
 149     # Do we have migrations
 150     should_run_sqam_migrations = db.engine.has_table("core__migrations") and \
 151                                  sqam_migrations_to_run(db, app_config,
 152                                                         global_config)
 153
 154     # Looks like a fresh database!
 155     # (We set up this variable here because doing "run_all_migrations" below
 156     # will change things.)
 157     fresh_database = (
 158         not db.engine.has_table("core__migrations") and
 159         not db.engine.has_table("alembic_version"))
 160
 161     # Run the migrations
 162     if should_run_sqam_migrations:
 163         run_all_migrations(db, app_config, global_config)
 164
 165     run_alembic_migrations(db, app_config, global_config)
 166
 167     # If this was our first time initializing the database,
 168     # we must lay down the foundations
 169     if fresh_database:
 170         run_foundations(db, global_config)
 171
 172
 173 def run_all_migrations(db, app_config, global_config):
 174     """Initialize or migrates a database.
 175
 176     Initializes or migrates a database that already has a
 177     connection setup and also initializes or migrates all
 178     extensions based on the config files.
 179
 180     It can be used to initialize an in-memory database for
 181     testing.
 182     """
 183     # Gather information from all media managers / projects
 184     dbdatas = gather_database_data(
 185         list(global_config.get('plugins', {}).keys()))
 186
 187     Session = sessionmaker(bind=db.engine)
 188
 189     # Setup media managers for all dbdata, run init/migrate and print info
 190     # For each component, create/migrate tables
 191     for dbdata in dbdatas:
 192         migration_manager = dbdata.make_migration_manager(Session())
 193         migration_manager.init_or_migrate()
 194
 195
 196 def sqam_migrations_to_run(db, app_config, global_config):
 197     """
 198     Check whether any plugins have sqlalchemy-migrate migrations left to run.
 199
 200     This is a kludge so we can transition away from sqlalchemy-migrate
 201     except where necessary.
 202     """
 203     # @@: This shares a lot of code with run_all_migrations, but both
 204     # are legacy and will be removed at some point.
 205
 206     # Gather information from all media managers / projects
 207     dbdatas = gather_database_data(
 208         list(global_config.get('plugins', {}).keys()))
 209
 210     Session = sessionmaker(bind=db.engine)
 211
 212     # We can bail out early if it turns out that sqlalchemy-migrate
 213     # was never installed with any migrations
 214     from mediagoblin.db.models import MigrationData
 215     if Session().query(MigrationData).filter_by(
 216             name=u"__main__").first() is None:
 217         return False
 218
 219     # Setup media managers for all dbdata, run init/migrate and print info
 220     # For each component, create/migrate tables
 221     for dbdata in dbdatas:
 222         migration_manager = dbdata.make_migration_manager(Session())
 223         if migration_manager.migrations_to_run():
 224             # If *any* migration managers have migrations to run,
 225             # we'll have to run them.
 226             return True
 227
 228     # Otherwise, scot free!
 229     return False
 230
 231
 232 def dbupdate(args):
 233     global_config, app_config = setup_global_and_app_config(args.conf_file)
 234     run_dbupdate(app_config, global_config)