mediagoblin/gmg_commands/dbupdate.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import logging
  18
  19 import six
  20 from alembic import command
  21 from sqlalchemy.orm import sessionmaker
  22
  23 from mediagoblin.db.open import setup_connection_and_db_from_config
  24 from mediagoblin.db.migration_tools import (
  25     MigrationManager, build_alembic_config, populate_table_foundations)
  26 from mediagoblin.init import setup_global_and_app_config
  27 from mediagoblin.tools.common import import_component
  28
  29 _log = logging.getLogger(__name__)
  30 logging.basicConfig()
  31 ## Let's not set the level as debug by default to avoid confusing users :)
  32 # _log.setLevel(logging.DEBUG)
  33
  34
  35 def dbupdate_parse_setup(subparser):
  36     pass
  37
  38
  39 class DatabaseData(object):
  40     def __init__(self, name, models, migrations):
  41         self.name = name
  42         self.models = models
  43         self.migrations = migrations
  44
  45     def make_migration_manager(self, session):
  46         return MigrationManager(
  47             self.name, self.models, self.migrations, session)
  48
  49
  50 def gather_database_data(plugins):
  51     """
  52     Gather all database data relevant to the extensions installed.
  53
  54     Gather all database data relevant to the extensions we have
  55     installed so we can do migrations and table initialization.
  56
  57     Returns a list of DatabaseData objects.
  58     """
  59     managed_dbdata = []
  60
  61     # Add main first
  62     from mediagoblin.db.models import MODELS as MAIN_MODELS
  63     from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS
  64
  65     managed_dbdata.append(
  66         DatabaseData(
  67             u'__main__', MAIN_MODELS, MAIN_MIGRATIONS))
  68
  69     for plugin in plugins:
  70         try:
  71             models = import_component('{0}.models:MODELS'.format(plugin))
  72         except ImportError as exc:
  73             _log.debug('No models found for {0}: {1}'.format(
  74                 plugin,
  75                 exc))
  76
  77             models = []
  78         except AttributeError as exc:
  79             _log.warning('Could not find MODELS in {0}.models, have you '
  80                          'forgotten to add it? ({1})'.format(plugin, exc))
  81             models = []
  82
  83         try:
  84             migrations = import_component('{0}.migrations:MIGRATIONS'.format(
  85                 plugin))
  86         except ImportError as exc:
  87             _log.debug('No migrations found for {0}: {1}'.format(
  88                 plugin,
  89                 exc))
  90
  91             migrations = {}
  92         except AttributeError as exc:
  93             _log.debug('Could not find MIGRATIONS in {0}.migrations, have you'
  94                        'forgotten to add it? ({1})'.format(plugin, exc))
  95             migrations = {}
  96
  97         if models:
  98             managed_dbdata.append(
  99                 DatabaseData(plugin, models, migrations))
 100
 101     return managed_dbdata
 102
 103
 104 def run_foundations(db, global_config):
 105     """
 106     Gather foundations data and run it.
 107     """
 108     from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
 109     all_foundations = [(u"__main__", MAIN_FOUNDATIONS)]
 110
 111     Session = sessionmaker(bind=db.engine)
 112     session = Session()
 113
 114     plugins = global_config.get('plugins', {})
 115
 116     for plugin in plugins:
 117         try:
 118             foundations = import_component(
 119                 '{0}.models:FOUNDATIONS'.format(plugin))
 120             all_foundations.append((plugin, foundations))
 121         except ImportError as exc:
 122             continue
 123         except AttributeError as exc:
 124             continue
 125
 126     for name, foundations in all_foundations:
 127         populate_table_foundations(session, foundations, name)
 128
 129
 130 def run_alembic_migrations(db, app_config, global_config):
 131     """Initialize a database and runs all Alembic migrations."""
 132     Session = sessionmaker(bind=db.engine)
 133     session = Session()
 134     cfg = build_alembic_config(global_config, None, session)
 135
 136     res = command.upgrade(cfg, 'heads')
 137     session.commit()
 138     return res
 139
 140
 141 def run_dbupdate(app_config, global_config):
 142     """
 143     Initialize or migrate the database as specified by the config file.
 144
 145     Will also initialize or migrate all extensions (media types, and
 146     in the future, plugins)
 147     """
 148     # Set up the database
 149     db = setup_connection_and_db_from_config(app_config, migrations=True)
 150
 151     # Do we have migrations
 152     should_run_sqam_migrations = db.engine.has_table("core__migrations") and \
 153                                  sqam_migrations_to_run(db, app_config,
 154                                                         global_config)
 155
 156     # Looks like a fresh database!
 157     # (We set up this variable here because doing "run_all_migrations" below
 158     # will change things.)
 159     fresh_database = (
 160         not db.engine.has_table("core__migrations") and
 161         not db.engine.has_table("alembic_version"))
 162
 163     # Run the migrations
 164     if should_run_sqam_migrations:
 165         run_all_migrations(db, app_config, global_config)
 166
 167     run_alembic_migrations(db, app_config, global_config)
 168
 169     # If this was our first time initializing the database,
 170     # we must lay down the foundations
 171     if fresh_database:
 172         run_foundations(db, global_config)
 173
 174
 175 def run_all_migrations(db, app_config, global_config):
 176     """Initialize or migrates a database.
 177
 178     Initializes or migrates a database that already has a
 179     connection setup and also initializes or migrates all
 180     extensions based on the config files.
 181
 182     It can be used to initialize an in-memory database for
 183     testing.
 184     """
 185     # Gather information from all media managers / projects
 186     dbdatas = gather_database_data(
 187         list(global_config.get('plugins', {}).keys()))
 188
 189     Session = sessionmaker(bind=db.engine)
 190
 191     # Setup media managers for all dbdata, run init/migrate and print info
 192     # For each component, create/migrate tables
 193     for dbdata in dbdatas:
 194         migration_manager = dbdata.make_migration_manager(Session())
 195         migration_manager.init_or_migrate()
 196
 197
 198 def sqam_migrations_to_run(db, app_config, global_config):
 199     """
 200     Check whether any plugins have sqlalchemy-migrate migrations left to run.
 201
 202     This is a kludge so we can transition away from sqlalchemy-migrate
 203     except where necessary.
 204     """
 205     # @@: This shares a lot of code with run_all_migrations, but both
 206     # are legacy and will be removed at some point.
 207
 208     # Gather information from all media managers / projects
 209     dbdatas = gather_database_data(
 210         list(global_config.get('plugins', {}).keys()))
 211
 212     Session = sessionmaker(bind=db.engine)
 213
 214     # We can bail out early if it turns out that sqlalchemy-migrate
 215     # was never installed with any migrations
 216     from mediagoblin.db.models import MigrationData
 217     if Session().query(MigrationData).filter_by(
 218             name=u"__main__").first() is None:
 219         return False
 220
 221     # Setup media managers for all dbdata, run init/migrate and print info
 222     # For each component, create/migrate tables
 223     for dbdata in dbdatas:
 224         migration_manager = dbdata.make_migration_manager(Session())
 225         if migration_manager.migrations_to_run():
 226             # If *any* migration managers have migrations to run,
 227             # we'll have to run them.
 228             return True
 229
 230     # Otherwise, scot free!
 231     return False
 232
 233
 234 def dbupdate(args):
 235     global_config, app_config = setup_global_and_app_config(args.conf_file)
 236     run_dbupdate(app_config, global_config)