mediagoblin/db/sql/convert.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from copy import copy
  18 from itertools import chain, imap
  19
  20 from mediagoblin.init import setup_global_and_app_config
  21
  22 from mediagoblin.db.sql.base import Session
  23 from mediagoblin.db.sql.models_v0 import Base_v0
  24 from mediagoblin.db.sql.models_v0 import (User, MediaEntry, MediaComment,
  25     Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData,
  26     ImageData, VideoData, AsciiData, AudioData)
  27 from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
  28     sql_connect
  29 from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \
  30     mongo_connect
  31
  32
  33 obj_id_table = dict()
  34
  35
  36 def add_obj_ids(entry, new_entry):
  37     global obj_id_table
  38     print "\t%r -> SQL id %r" % (entry._id, new_entry.id)
  39     obj_id_table[entry._id] = new_entry.id
  40
  41
  42 def copy_attrs(entry, new_entry, attr_list):
  43     for a in attr_list:
  44         val = entry[a]
  45         setattr(new_entry, a, val)
  46
  47
  48 def copy_reference_attr(entry, new_entry, ref_attr):
  49     val = entry[ref_attr]
  50     val = obj_id_table[val]
  51     setattr(new_entry, ref_attr, val)
  52
  53
  54 def convert_users(mk_db):
  55     session = Session()
  56
  57     for entry in mk_db.User.find().sort('created'):
  58         print entry.username
  59
  60         new_entry = User()
  61         copy_attrs(entry, new_entry,
  62             ('username', 'email', 'created', 'pw_hash', 'email_verified',
  63              'status', 'verification_key', 'is_admin', 'url',
  64              'bio',
  65              'fp_verification_key', 'fp_token_expire',))
  66         # new_entry.fp_verification_expire = entry.fp_token_expire
  67
  68         session.add(new_entry)
  69         session.flush()
  70         add_obj_ids(entry, new_entry)
  71
  72     session.commit()
  73     session.close()
  74
  75
  76 def convert_media_entries(mk_db):
  77     session = Session()
  78
  79     for entry in mk_db.MediaEntry.find().sort('created'):
  80         print repr(entry.title)
  81
  82         new_entry = MediaEntry()
  83         copy_attrs(entry, new_entry,
  84             ('title', 'slug', 'created',
  85              'description',
  86              'media_type', 'state', 'license',
  87              'fail_error', 'fail_metadata',
  88              'queued_task_id',))
  89         copy_reference_attr(entry, new_entry, "uploader")
  90
  91         session.add(new_entry)
  92         session.flush()
  93         add_obj_ids(entry, new_entry)
  94
  95         for key, value in entry.media_files.iteritems():
  96             new_file = MediaFile(name=key, file_path=value)
  97             new_file.media_entry = new_entry.id
  98             Session.add(new_file)
  99
 100         for attachment in entry.attachment_files:
 101             new_attach = MediaAttachmentFile(
 102                 name=attachment["name"],
 103                 filepath=attachment["filepath"],
 104                 created=attachment["created"]
 105                 )
 106             new_attach.media_entry = new_entry.id
 107             Session.add(new_attach)
 108
 109     session.commit()
 110     session.close()
 111
 112
 113 def convert_image(mk_db):
 114     session = Session()
 115
 116     for media in mk_db.MediaEntry.find(
 117             {'media_type': 'mediagoblin.media_types.image'}).sort('created'):
 118         media_data = copy(media.media_data)
 119
 120         if len(media_data):
 121             media_data_row = ImageData(**media_data)
 122             media_data_row.media_entry = obj_id_table[media['_id']]
 123             session.add(media_data_row)
 124
 125     session.commit()
 126     session.close()
 127
 128
 129 def convert_video(mk_db):
 130     session = Session()
 131
 132     for media in mk_db.MediaEntry.find(
 133             {'media_type': 'mediagoblin.media_types.video'}).sort('created'):
 134         media_data_row = VideoData(**media.media_data)
 135         media_data_row.media_entry = obj_id_table[media['_id']]
 136         session.add(media_data_row)
 137
 138     session.commit()
 139     session.close()
 140
 141
 142 def convert_media_tags(mk_db):
 143     session = Session()
 144     session.autoflush = False
 145
 146     for media in mk_db.MediaEntry.find().sort('created'):
 147         print repr(media.title)
 148
 149         for otag in media.tags:
 150             print "  ", repr((otag["slug"], otag["name"]))
 151
 152             nslug = session.query(Tag).filter_by(slug=otag["slug"]).first()
 153             print "     ", repr(nslug)
 154             if nslug is None:
 155                 nslug = Tag(slug=otag["slug"])
 156                 session.add(nslug)
 157                 session.flush()
 158             print "     ", repr(nslug), nslug.id
 159
 160             ntag = MediaTag()
 161             ntag.tag = nslug.id
 162             ntag.name = otag["name"]
 163             ntag.media_entry = obj_id_table[media._id]
 164             session.add(ntag)
 165
 166     session.commit()
 167     session.close()
 168
 169
 170 def convert_media_comments(mk_db):
 171     session = Session()
 172
 173     for entry in mk_db.MediaComment.find().sort('created'):
 174         print repr(entry.content)
 175
 176         new_entry = MediaComment()
 177         copy_attrs(entry, new_entry,
 178             ('created',
 179              'content',))
 180
 181         try:
 182             copy_reference_attr(entry, new_entry, "media_entry")
 183             copy_reference_attr(entry, new_entry, "author")
 184         except KeyError as e:
 185             print('KeyError in convert_media_comments(): {0}'.format(e))
 186         else:
 187             session.add(new_entry)
 188             session.flush()
 189             add_obj_ids(entry, new_entry)
 190
 191     session.commit()
 192     session.close()
 193
 194
 195 media_types_tables = (
 196     ("mediagoblin.media_types.image", (ImageData,)),
 197     ("mediagoblin.media_types.video", (VideoData,)),
 198     ("mediagoblin.media_types.ascii", (AsciiData,)),
 199     ("mediagoblin.media_types.audio", (AudioData,)),
 200     )
 201
 202
 203 def convert_add_migration_versions(dummy_sql_db):
 204     session = Session()
 205
 206     for name in chain(("__main__",),
 207                       imap(lambda e: e[0], media_types_tables)):
 208         print "\tAdding %s" % (name,)
 209         m = MigrationData(name=unicode(name), version=0)
 210         session.add(m)
 211
 212     session.commit()
 213     session.close()
 214
 215
 216 def cleanup_sql_tables(sql_db):
 217     for mt, table_list in media_types_tables:
 218         session = Session()
 219
 220         count = session.query(MediaEntry.media_type). \
 221             filter_by(media_type=unicode(mt)).count()
 222         print "  %s: %d entries" % (mt, count)
 223         if count == 0:
 224             print "\tRemoving migration info"
 225             mi = session.query(MigrationData).filter_by(name=unicode(mt)).one()
 226             session.delete(mi)
 227             session.commit()
 228             session.close()
 229
 230             print "\tDropping tables %r" % (table_list,)
 231             tables = [model.__table__ for model in table_list]
 232             Base_v0.metadata.drop_all(sql_db.engine, tables=tables)
 233
 234         session.close()
 235
 236
 237 def print_header(title):
 238     print "\n=== %s ===" % (title,)
 239
 240
 241 convert_call_list = (
 242     ("Converting Users", convert_users),
 243     ("Converting Media Entries", convert_media_entries),
 244     ("Converting Media Data for Images", convert_image),
 245     ("Cnnverting Media Data for Videos", convert_video),
 246     ("Converting Tags for Media", convert_media_tags),
 247     ("Converting Media Comments", convert_media_comments),
 248     )
 249
 250 sql_call_list = (
 251     ("Filling Migration Tables", convert_add_migration_versions),
 252     ("Analyzing/Cleaning SQL Data", cleanup_sql_tables),
 253     )
 254
 255 def run_conversion(config_name):
 256     global_config, app_config = setup_global_and_app_config(config_name)
 257
 258     sql_conn, sql_db = sql_connect(app_config)
 259     mk_conn, mk_db = mongo_connect(app_config)
 260
 261     Base_v0.metadata.create_all(sql_db.engine)
 262
 263     for title, func in convert_call_list:
 264         print_header(title)
 265         func(mk_db)
 266         Session.remove()
 267
 268     for title, func in sql_call_list:
 269         print_header(title)
 270         func(sql_db)
 271         Session.remove()
 272
 273
 274 if __name__ == '__main__':
 275     run_conversion("mediagoblin.ini")