From 1e46dc2537c5ee706a55876c4dbf86129baafbbf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Nov 2012 09:49:45 +0100 Subject: [PATCH] Move db.sql.util to db.util Now that sqlalchemy is providing the database abstractions, there is no need to hide everything in db.sql. sub-modules. It complicates the code and provides a futher layer of indirection. Move the db.sql.util.py to db.util.py and adapt the importers. --- mediagoblin/db/migrations.py | 2 +- mediagoblin/db/sql/util.py | 327 ----------------------- mediagoblin/db/util.py | 312 ++++++++++++++++++++- mediagoblin/gmg_commands/dbupdate.py | 2 +- mediagoblin/plugins/oauth/migrations.py | 2 +- mediagoblin/tests/test_sql_migrations.py | 2 +- 6 files changed, 313 insertions(+), 334 deletions(-) delete mode 100644 mediagoblin/db/sql/util.py diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 99448bfa..3f0cb6f0 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -23,7 +23,7 @@ from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.declarative import declarative_base from migrate.changeset.constraint import UniqueConstraint -from mediagoblin.db.sql.util import RegisterMigration +from mediagoblin.db.util import RegisterMigration from mediagoblin.db.sql.models import MediaEntry, Collection, User MIGRATIONS = {} diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/sql/util.py deleted file mode 100644 index e38ca1e6..00000000 --- a/mediagoblin/db/sql/util.py +++ /dev/null @@ -1,327 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -import sys -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection - -from mediagoblin.tools.common import simple_printer - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - - def __init__(self, name, models, migration_registry, session, - printer=simple_printer): - """ - Args: - - name: identifier of this section of the database - - session: session we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.name = unicode(name) - self.models = models - self.session = session - self.migration_registry = migration_registry - self._sorted_migrations = None - self.printer = printer - - # For convenience - from mediagoblin.db.sql.models import MigrationData - - self.migration_model = MigrationData - self.migration_table = MigrationData.__table__ - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - @property - def migration_data(self): - """ - Get the migration row associated with this object, if any. - """ - return self.session.query( - self.migration_model).filter_by(name=self.name).first() - - @property - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - @property - def database_current_migration(self): - """ - Return the current migration in the database. - """ - # If the table doesn't even exist, return None. - if not self.migration_table.exists(self.session.bind): - return None - - # Also return None if self.migration_data is None. - if self.migration_data is None: - return None - - return self.migration_data.version - - def set_current_migration(self, migration_number=None): - """ - Set the migration in the database to migration_number - (or, the latest available) - """ - self.migration_data.version = migration_number or self.latest_migration - self.session.commit() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that this will fail if there's no migration record for - this class! - """ - assert self.database_current_migration is not None - - db_current_migration = self.database_current_migration - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - - def init_tables(self): - """ - Create all tables relative to this package - """ - # sanity check before we proceed, none of these should be created - for model in self.models: - # Maybe in the future just print out a "Yikes!" or something? - assert not model.__table__.exists(self.session.bind) - - self.migration_model.metadata.create_all( - self.session.bind, - tables=[model.__table__ for model in self.models]) - - def create_new_migration_record(self): - """ - Create a new migration record for this migration set - """ - migration_record = self.migration_model( - name=self.name, - version=self.latest_migration) - self.session.add(migration_record) - self.session.commit() - - def dry_run(self): - """ - Print out a dry run of what we would have upgraded. - """ - if self.database_current_migration is None: - self.printer( - u'~> Woulda initialized: %s\n' % self.name_for_printing()) - return u'inited' - - migrations_to_run = self.migrations_to_run() - if migrations_to_run: - self.printer( - u'~> Woulda updated %s:\n' % self.name_for_printing()) - - for migration_number, migration_func in migrations_to_run(): - self.printer( - u' + Would update %s, "%s"\n' % ( - migration_number, migration_func.func_name)) - - return u'migrated' - - def name_for_printing(self): - if self.name == u'__main__': - return u"main mediagoblin tables" - else: - # TODO: Use the friendlier media manager "human readable" name - return u'media type "%s"' % self.name - - def init_or_migrate(self): - """ - Initialize the database or migrate if appropriate. - - Returns information about whether or not we initialized - ('inited'), migrated ('migrated'), or did nothing (None) - """ - assure_migrations_table_setup(self.session) - - # Find out what migration number, if any, this database data is at, - # and what the latest is. - migration_number = self.database_current_migration - - # Is this our first time? Is there even a table entry for - # this identifier? - # If so: - # - create all tables - # - create record in migrations registry - # - print / inform the user - # - return 'inited' - if migration_number is None: - self.printer(u"-> Initializing %s... " % self.name_for_printing()) - - self.init_tables() - # auto-set at latest migration number - self.create_new_migration_record() - - self.printer(u"done.\n") - self.set_current_migration() - return u'inited' - - # Run migrations, if appropriate. - migrations_to_run = self.migrations_to_run() - if migrations_to_run: - self.printer( - u'-> Updating %s:\n' % self.name_for_printing()) - for migration_number, migration_func in migrations_to_run: - self.printer( - u' + Running migration %s, "%s"... ' % ( - migration_number, migration_func.func_name)) - migration_func(self.session) - self.set_current_migration(migration_number) - self.printer('done.\n') - - return u'migrated' - - # Otherwise return None. Well it would do this anyway, but - # for clarity... ;) - return None - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry): - assert migration_number > 0, "Migration number must be > 0!" - assert migration_number not in migration_registry, \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -def assure_migrations_table_setup(db): - """ - Make sure the migrations table is set up in the database. - """ - from mediagoblin.db.sql.models import MigrationData - - if not MigrationData.__table__.exists(db.bind): - MigrationData.metadata.create_all( - db.bind, tables=[MigrationData.__table__]) - - -########################## -# Random utility functions -########################## - - -def atomic_update(table, query_dict, update_values): - table.find(query_dict).update(update_values, - synchronize_session=False) - Session.commit() - - -def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): - filt = (MediaEntry.uploader == uploader_id) \ - & (MediaEntry.slug == slug) - if ignore_m_id is not None: - filt = filt & (MediaEntry.id != ignore_m_id) - does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None - return does_exist - - -def media_entries_for_tag_slug(dummy_db, tag_slug): - return MediaEntry.query \ - .join(MediaEntry.tags_helper) \ - .join(MediaTag.tag_helper) \ - .filter( - (MediaEntry.state == u'processed') - & (Tag.slug == tag_slug)) - - -def clean_orphan_tags(commit=True): - """Search for unused MediaTags and delete them""" - q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) - for t in q1: - Session.delete(t) - # The "let the db do all the work" version: - # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) - # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) - # q2.delete(synchronize_session = False) - if commit: - Session.commit() - - -def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id): - filt = (Collection.creator == creator_id) \ - & (Collection.slug == slug) - if ignore_c_id is not None: - filt = filt & (Collection.id != ignore_c_id) - does_exist = Session.query(Collection.id).filter(filt).first() is not None - return does_exist - - -if __name__ == '__main__': - from mediagoblin.db.open import setup_connection_and_db_from_config - - db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'}) - - clean_orphan_tags() diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index ef3abf9b..2f167e45 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -13,7 +13,313 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import sys +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection -#TODO: check now after mongo removal if we can't rip out a layer of abstraction -from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \ - media_entries_for_tag_slug, check_collection_slug_used +from mediagoblin.tools.common import simple_printer + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + + def __init__(self, name, models, migration_registry, session, + printer=simple_printer): + """ + Args: + - name: identifier of this section of the database + - session: session we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.name = unicode(name) + self.models = models + self.session = session + self.migration_registry = migration_registry + self._sorted_migrations = None + self.printer = printer + + # For convenience + from mediagoblin.db.sql.models import MigrationData + + self.migration_model = MigrationData + self.migration_table = MigrationData.__table__ + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + @property + def migration_data(self): + """ + Get the migration row associated with this object, if any. + """ + return self.session.query( + self.migration_model).filter_by(name=self.name).first() + + @property + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + @property + def database_current_migration(self): + """ + Return the current migration in the database. + """ + # If the table doesn't even exist, return None. + if not self.migration_table.exists(self.session.bind): + return None + + # Also return None if self.migration_data is None. + if self.migration_data is None: + return None + + return self.migration_data.version + + def set_current_migration(self, migration_number=None): + """ + Set the migration in the database to migration_number + (or, the latest available) + """ + self.migration_data.version = migration_number or self.latest_migration + self.session.commit() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that this will fail if there's no migration record for + this class! + """ + assert self.database_current_migration is not None + + db_current_migration = self.database_current_migration + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + + def init_tables(self): + """ + Create all tables relative to this package + """ + # sanity check before we proceed, none of these should be created + for model in self.models: + # Maybe in the future just print out a "Yikes!" or something? + assert not model.__table__.exists(self.session.bind) + + self.migration_model.metadata.create_all( + self.session.bind, + tables=[model.__table__ for model in self.models]) + + def create_new_migration_record(self): + """ + Create a new migration record for this migration set + """ + migration_record = self.migration_model( + name=self.name, + version=self.latest_migration) + self.session.add(migration_record) + self.session.commit() + + def dry_run(self): + """ + Print out a dry run of what we would have upgraded. + """ + if self.database_current_migration is None: + self.printer( + u'~> Woulda initialized: %s\n' % self.name_for_printing()) + return u'inited' + + migrations_to_run = self.migrations_to_run() + if migrations_to_run: + self.printer( + u'~> Woulda updated %s:\n' % self.name_for_printing()) + + for migration_number, migration_func in migrations_to_run(): + self.printer( + u' + Would update %s, "%s"\n' % ( + migration_number, migration_func.func_name)) + + return u'migrated' + + def name_for_printing(self): + if self.name == u'__main__': + return u"main mediagoblin tables" + else: + # TODO: Use the friendlier media manager "human readable" name + return u'media type "%s"' % self.name + + def init_or_migrate(self): + """ + Initialize the database or migrate if appropriate. + + Returns information about whether or not we initialized + ('inited'), migrated ('migrated'), or did nothing (None) + """ + assure_migrations_table_setup(self.session) + + # Find out what migration number, if any, this database data is at, + # and what the latest is. + migration_number = self.database_current_migration + + # Is this our first time? Is there even a table entry for + # this identifier? + # If so: + # - create all tables + # - create record in migrations registry + # - print / inform the user + # - return 'inited' + if migration_number is None: + self.printer(u"-> Initializing %s... " % self.name_for_printing()) + + self.init_tables() + # auto-set at latest migration number + self.create_new_migration_record() + + self.printer(u"done.\n") + self.set_current_migration() + return u'inited' + + # Run migrations, if appropriate. + migrations_to_run = self.migrations_to_run() + if migrations_to_run: + self.printer( + u'-> Updating %s:\n' % self.name_for_printing()) + for migration_number, migration_func in migrations_to_run: + self.printer( + u' + Running migration %s, "%s"... ' % ( + migration_number, migration_func.func_name)) + migration_func(self.session) + self.set_current_migration(migration_number) + self.printer('done.\n') + + return u'migrated' + + # Otherwise return None. Well it would do this anyway, but + # for clarity... ;) + return None + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry): + assert migration_number > 0, "Migration number must be > 0!" + assert migration_number not in migration_registry, \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +def assure_migrations_table_setup(db): + """ + Make sure the migrations table is set up in the database. + """ + from mediagoblin.db.sql.models import MigrationData + + if not MigrationData.__table__.exists(db.bind): + MigrationData.metadata.create_all( + db.bind, tables=[MigrationData.__table__]) + + +########################## +# Random utility functions +########################## + + +def atomic_update(table, query_dict, update_values): + table.find(query_dict).update(update_values, + synchronize_session=False) + Session.commit() + + +def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): + filt = (MediaEntry.uploader == uploader_id) \ + & (MediaEntry.slug == slug) + if ignore_m_id is not None: + filt = filt & (MediaEntry.id != ignore_m_id) + does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None + return does_exist + + +def media_entries_for_tag_slug(dummy_db, tag_slug): + return MediaEntry.query \ + .join(MediaEntry.tags_helper) \ + .join(MediaTag.tag_helper) \ + .filter( + (MediaEntry.state == u'processed') + & (Tag.slug == tag_slug)) + + +def clean_orphan_tags(commit=True): + """Search for unused MediaTags and delete them""" + q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) + for t in q1: + Session.delete(t) + # The "let the db do all the work" version: + # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) + # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) + # q2.delete(synchronize_session = False) + if commit: + Session.commit() + + +def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id): + filt = (Collection.creator == creator_id) \ + & (Collection.slug == slug) + if ignore_c_id is not None: + filt = filt & (Collection.id != ignore_c_id) + does_exist = Session.query(Collection.id).filter(filt).first() is not None + return does_exist + + +if __name__ == '__main__': + from mediagoblin.db.open import setup_connection_and_db_from_config + + db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'}) + + clean_orphan_tags() diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 2c34bb51..95898c08 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -19,7 +19,7 @@ import logging from sqlalchemy.orm import sessionmaker from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.db.sql.util import MigrationManager +from mediagoblin.db.util import MigrationManager from mediagoblin.init import setup_global_and_app_config from mediagoblin.tools.common import import_component diff --git a/mediagoblin/plugins/oauth/migrations.py b/mediagoblin/plugins/oauth/migrations.py index 797e7585..3beef087 100644 --- a/mediagoblin/plugins/oauth/migrations.py +++ b/mediagoblin/plugins/oauth/migrations.py @@ -19,7 +19,7 @@ from sqlalchemy import (MetaData, Table, Column, Integer, Unicode, Enum, DateTime, ForeignKey) from sqlalchemy.ext.declarative import declarative_base -from mediagoblin.db.sql.util import RegisterMigration +from mediagoblin.db.util import RegisterMigration from mediagoblin.db.sql.models import User diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index 6383d096..0e7102bc 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -26,7 +26,7 @@ from sqlalchemy.sql import select, insert from migrate import changeset from mediagoblin.db.sql.base import GMGTableBase -from mediagoblin.db.sql.util import MigrationManager, RegisterMigration +from mediagoblin.db.util import MigrationManager, RegisterMigration from mediagoblin.tools.common import CollectingPrinter -- 2.25.1