mediagoblin/db/models.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 """
  18 TODO: indexes on foreignkeys, where useful.
  19 """
  20
  21 import logging
  22 import datetime
  23
  24 from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \
  25         Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \
  26         SmallInteger
  27 from sqlalchemy.orm import relationship, backref, with_polymorphic
  28 from sqlalchemy.orm.collections import attribute_mapped_collection
  29 from sqlalchemy.sql.expression import desc
  30 from sqlalchemy.ext.associationproxy import association_proxy
  31 from sqlalchemy.util import memoized_property
  32
  33
  34 from mediagoblin.db.extratypes import PathTupleWithSlashes, JSONEncoded
  35 from mediagoblin.db.base import Base, DictReadAttrProxy
  36 from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, \
  37         MediaCommentMixin, CollectionMixin, CollectionItemMixin
  38 from mediagoblin.tools.files import delete_media_files
  39 from mediagoblin.tools.common import import_component
  40
  41 # It's actually kind of annoying how sqlalchemy-migrate does this, if
  42 # I understand it right, but whatever.  Anyway, don't remove this :P
  43 #
  44 # We could do migration calls more manually instead of relying on
  45 # this import-based meddling...
  46 from migrate import changeset
  47
  48 _log = logging.getLogger(__name__)
  49
  50
  51 class User(Base, UserMixin):
  52     """
  53     TODO: We should consider moving some rarely used fields
  54     into some sort of "shadow" table.
  55     """
  56     __tablename__ = "core__users"
  57
  58     id = Column(Integer, primary_key=True)
  59     username = Column(Unicode, nullable=False, unique=True)
  60     # Note: no db uniqueness constraint on email because it's not
  61     # reliable (many email systems case insensitive despite against
  62     # the RFC) and because it would be a mess to implement at this
  63     # point.
  64     email = Column(Unicode, nullable=False)
  65     pw_hash = Column(Unicode, nullable=False)
  66     email_verified = Column(Boolean, default=False)
  67     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
  68     status = Column(Unicode, default=u"needs_email_verification", nullable=False)
  69     # Intented to be nullable=False, but migrations would not work for it
  70     # set to nullable=True implicitly.
  71     wants_comment_notification = Column(Boolean, default=True)
  72     license_preference = Column(Unicode)
  73     is_admin = Column(Boolean, default=False, nullable=False)
  74     url = Column(Unicode)
  75     bio = Column(UnicodeText)  # ??
  76
  77     ## TODO
  78     # plugin data would be in a separate model
  79
  80     def __repr__(self):
  81         return '<{0} #{1} {2} {3} "{4}">'.format(
  82                 self.__class__.__name__,
  83                 self.id,
  84                 'verified' if self.email_verified else 'non-verified',
  85                 'admin' if self.is_admin else 'user',
  86                 self.username)
  87
  88     def delete(self, **kwargs):
  89         """Deletes a User and all related entries/comments/files/..."""
  90         # Collections get deleted by relationships.
  91
  92         media_entries = MediaEntry.query.filter(MediaEntry.uploader == self.id)
  93         for media in media_entries:
  94             # TODO: Make sure that "MediaEntry.delete()" also deletes
  95             # all related files/Comments
  96             media.delete(del_orphan_tags=False, commit=False)
  97
  98         # Delete now unused tags
  99         # TODO: import here due to cyclic imports!!! This cries for refactoring
 100         from mediagoblin.db.util import clean_orphan_tags
 101         clean_orphan_tags(commit=False)
 102
 103         # Delete user, pass through commit=False/True in kwargs
 104         super(User, self).delete(**kwargs)
 105         _log.info('Deleted user "{0}" account'.format(self.username))
 106
 107
 108 class MediaEntry(Base, MediaEntryMixin):
 109     """
 110     TODO: Consider fetching the media_files using join
 111     """
 112     __tablename__ = "core__media_entries"
 113
 114     id = Column(Integer, primary_key=True)
 115     uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
 116     title = Column(Unicode, nullable=False)
 117     slug = Column(Unicode)
 118     created = Column(DateTime, nullable=False, default=datetime.datetime.now,
 119         index=True)
 120     description = Column(UnicodeText) # ??
 121     media_type = Column(Unicode, nullable=False)
 122     state = Column(Unicode, default=u'unprocessed', nullable=False)
 123         # or use sqlalchemy.types.Enum?
 124     license = Column(Unicode)
 125     collected = Column(Integer, default=0)
 126
 127     fail_error = Column(Unicode)
 128     fail_metadata = Column(JSONEncoded)
 129
 130     transcoding_progress = Column(SmallInteger)
 131
 132     queued_media_file = Column(PathTupleWithSlashes)
 133
 134     queued_task_id = Column(Unicode)
 135
 136     __table_args__ = (
 137         UniqueConstraint('uploader', 'slug'),
 138         {})
 139
 140     get_uploader = relationship(User)
 141
 142     media_files_helper = relationship("MediaFile",
 143         collection_class=attribute_mapped_collection("name"),
 144         cascade="all, delete-orphan"
 145         )
 146     media_files = association_proxy('media_files_helper', 'file_path',
 147         creator=lambda k, v: MediaFile(name=k, file_path=v)
 148         )
 149
 150     attachment_files_helper = relationship("MediaAttachmentFile",
 151         cascade="all, delete-orphan",
 152         order_by="MediaAttachmentFile.created"
 153         )
 154     attachment_files = association_proxy("attachment_files_helper", "dict_view",
 155         creator=lambda v: MediaAttachmentFile(
 156             name=v["name"], filepath=v["filepath"])
 157         )
 158
 159     tags_helper = relationship("MediaTag",
 160         cascade="all, delete-orphan" # should be automatically deleted
 161         )
 162     tags = association_proxy("tags_helper", "dict_view",
 163         creator=lambda v: MediaTag(name=v["name"], slug=v["slug"])
 164         )
 165
 166     collections_helper = relationship("CollectionItem",
 167         cascade="all, delete-orphan"
 168         )
 169     collections = association_proxy("collections_helper", "in_collection")
 170
 171     ## TODO
 172     # fail_error
 173
 174     def get_comments(self, ascending=False):
 175         order_col = MediaComment.created
 176         if not ascending:
 177             order_col = desc(order_col)
 178         return self.all_comments.order_by(order_col)
 179
 180     def url_to_prev(self, urlgen):
 181         """get the next 'newer' entry by this user"""
 182         media = MediaEntry.query.filter(
 183             (MediaEntry.uploader == self.uploader)
 184             & (MediaEntry.state == u'processed')
 185             & (MediaEntry.id > self.id)).order_by(MediaEntry.id).first()
 186
 187         if media is not None:
 188             return media.url_for_self(urlgen)
 189
 190     def url_to_next(self, urlgen):
 191         """get the next 'older' entry by this user"""
 192         media = MediaEntry.query.filter(
 193             (MediaEntry.uploader == self.uploader)
 194             & (MediaEntry.state == u'processed')
 195             & (MediaEntry.id < self.id)).order_by(desc(MediaEntry.id)).first()
 196
 197         if media is not None:
 198             return media.url_for_self(urlgen)
 199
 200     @property
 201     def media_data(self):
 202         return getattr(self, self.media_data_ref)
 203
 204     def media_data_init(self, **kwargs):
 205         """
 206         Initialize or update the contents of a media entry's media_data row
 207         """
 208         media_data = self.media_data
 209
 210         if media_data is None:
 211             # Get the correct table:
 212             table = import_component(self.media_type + '.models:DATA_MODEL')
 213             # No media data, so actually add a new one
 214             media_data = table(**kwargs)
 215             # Get the relationship set up.
 216             media_data.get_media_entry = self
 217         else:
 218             # Update old media data
 219             for field, value in kwargs.iteritems():
 220                 setattr(media_data, field, value)
 221
 222     @memoized_property
 223     def media_data_ref(self):
 224         return import_component(self.media_type + '.models:BACKREF_NAME')
 225
 226     def __repr__(self):
 227         safe_title = self.title.encode('ascii', 'replace')
 228
 229         return '<{classname} {id}: {title}>'.format(
 230                 classname=self.__class__.__name__,
 231                 id=self.id,
 232                 title=safe_title)
 233
 234     def delete(self, del_orphan_tags=True, **kwargs):
 235         """Delete MediaEntry and all related files/attachments/comments
 236
 237         This will *not* automatically delete unused collections, which
 238         can remain empty...
 239
 240         :param del_orphan_tags: True/false if we delete unused Tags too
 241         :param commit: True/False if this should end the db transaction"""
 242         # User's CollectionItems are automatically deleted via "cascade".
 243         # Comments on this Media are deleted by cascade, hopefully.
 244
 245         # Delete all related files/attachments
 246         try:
 247             delete_media_files(self)
 248         except OSError, error:
 249             # Returns list of files we failed to delete
 250             _log.error('No such files from the user "{1}" to delete: '
 251                        '{0}'.format(str(error), self.get_uploader))
 252         _log.info('Deleted Media entry id "{0}"'.format(self.id))
 253         # Related MediaTag's are automatically cleaned, but we might
 254         # want to clean out unused Tag's too.
 255         if del_orphan_tags:
 256             # TODO: Import here due to cyclic imports!!!
 257             #       This cries for refactoring
 258             from mediagoblin.db.util import clean_orphan_tags
 259             clean_orphan_tags(commit=False)
 260         # pass through commit=False/True in kwargs
 261         super(MediaEntry, self).delete(**kwargs)
 262
 263
 264 class FileKeynames(Base):
 265     """
 266     keywords for various places.
 267     currently the MediaFile keys
 268     """
 269     __tablename__ = "core__file_keynames"
 270     id = Column(Integer, primary_key=True)
 271     name = Column(Unicode, unique=True)
 272
 273     def __repr__(self):
 274         return "<FileKeyname %r: %r>" % (self.id, self.name)
 275
 276     @classmethod
 277     def find_or_new(cls, name):
 278         t = cls.query.filter_by(name=name).first()
 279         if t is not None:
 280             return t
 281         return cls(name=name)
 282
 283
 284 class MediaFile(Base):
 285     """
 286     TODO: Highly consider moving "name" into a new table.
 287     TODO: Consider preloading said table in software
 288     """
 289     __tablename__ = "core__mediafiles"
 290
 291     media_entry = Column(
 292         Integer, ForeignKey(MediaEntry.id),
 293         nullable=False)
 294     name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False)
 295     file_path = Column(PathTupleWithSlashes)
 296
 297     __table_args__ = (
 298         PrimaryKeyConstraint('media_entry', 'name_id'),
 299         {})
 300
 301     def __repr__(self):
 302         return "<MediaFile %s: %r>" % (self.name, self.file_path)
 303
 304     name_helper = relationship(FileKeynames, lazy="joined", innerjoin=True)
 305     name = association_proxy('name_helper', 'name',
 306         creator=FileKeynames.find_or_new
 307         )
 308
 309
 310 class MediaAttachmentFile(Base):
 311     __tablename__ = "core__attachment_files"
 312
 313     id = Column(Integer, primary_key=True)
 314     media_entry = Column(
 315         Integer, ForeignKey(MediaEntry.id),
 316         nullable=False)
 317     name = Column(Unicode, nullable=False)
 318     filepath = Column(PathTupleWithSlashes)
 319     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 320
 321     @property
 322     def dict_view(self):
 323         """A dict like view on this object"""
 324         return DictReadAttrProxy(self)
 325
 326
 327 class Tag(Base):
 328     __tablename__ = "core__tags"
 329
 330     id = Column(Integer, primary_key=True)
 331     slug = Column(Unicode, nullable=False, unique=True)
 332
 333     def __repr__(self):
 334         return "<Tag %r: %r>" % (self.id, self.slug)
 335
 336     @classmethod
 337     def find_or_new(cls, slug):
 338         t = cls.query.filter_by(slug=slug).first()
 339         if t is not None:
 340             return t
 341         return cls(slug=slug)
 342
 343
 344 class MediaTag(Base):
 345     __tablename__ = "core__media_tags"
 346
 347     id = Column(Integer, primary_key=True)
 348     media_entry = Column(
 349         Integer, ForeignKey(MediaEntry.id),
 350         nullable=False, index=True)
 351     tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True)
 352     name = Column(Unicode)
 353     # created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 354
 355     __table_args__ = (
 356         UniqueConstraint('tag', 'media_entry'),
 357         {})
 358
 359     tag_helper = relationship(Tag)
 360     slug = association_proxy('tag_helper', 'slug',
 361         creator=Tag.find_or_new
 362         )
 363
 364     def __init__(self, name=None, slug=None):
 365         Base.__init__(self)
 366         if name is not None:
 367             self.name = name
 368         if slug is not None:
 369             self.tag_helper = Tag.find_or_new(slug)
 370
 371     @property
 372     def dict_view(self):
 373         """A dict like view on this object"""
 374         return DictReadAttrProxy(self)
 375
 376
 377 class MediaComment(Base, MediaCommentMixin):
 378     __tablename__ = "core__media_comments"
 379
 380     id = Column(Integer, primary_key=True)
 381     media_entry = Column(
 382         Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
 383     author = Column(Integer, ForeignKey(User.id), nullable=False)
 384     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 385     content = Column(UnicodeText, nullable=False)
 386
 387     # Cascade: Comments are owned by their creator. So do the full thing.
 388     # lazy=dynamic: People might post a *lot* of comments,
 389     #     so make the "posted_comments" a query-like thing.
 390     get_author = relationship(User,
 391                               backref=backref("posted_comments",
 392                                               lazy="dynamic",
 393                                               cascade="all, delete-orphan"))
 394     get_entry = relationship(MediaEntry,
 395                              backref=backref("comments",
 396                                              lazy="dynamic",
 397                                              cascade="all, delete-orphan"))
 398
 399     # Cascade: Comments are somewhat owned by their MediaEntry.
 400     #     So do the full thing.
 401     # lazy=dynamic: MediaEntries might have many comments,
 402     #     so make the "all_comments" a query-like thing.
 403     get_media_entry = relationship(MediaEntry,
 404                                    backref=backref("all_comments",
 405                                                    lazy="dynamic",
 406                                                    cascade="all, delete-orphan"))
 407
 408
 409 class Collection(Base, CollectionMixin):
 410     """An 'album' or 'set' of media by a user.
 411
 412     On deletion, contained CollectionItems get automatically reaped via
 413     SQL cascade"""
 414     __tablename__ = "core__collections"
 415
 416     id = Column(Integer, primary_key=True)
 417     title = Column(Unicode, nullable=False)
 418     slug = Column(Unicode)
 419     created = Column(DateTime, nullable=False, default=datetime.datetime.now,
 420                      index=True)
 421     description = Column(UnicodeText)
 422     creator = Column(Integer, ForeignKey(User.id), nullable=False)
 423     # TODO: No of items in Collection. Badly named, can we migrate to num_items?
 424     items = Column(Integer, default=0)
 425
 426     # Cascade: Collections are owned by their creator. So do the full thing.
 427     get_creator = relationship(User,
 428                                backref=backref("collections",
 429                                                cascade="all, delete-orphan"))
 430
 431     __table_args__ = (
 432         UniqueConstraint('creator', 'slug'),
 433         {})
 434
 435     def get_collection_items(self, ascending=False):
 436         #TODO, is this still needed with self.collection_items being available?
 437         order_col = CollectionItem.position
 438         if not ascending:
 439             order_col = desc(order_col)
 440         return CollectionItem.query.filter_by(
 441             collection=self.id).order_by(order_col)
 442
 443
 444 class CollectionItem(Base, CollectionItemMixin):
 445     __tablename__ = "core__collection_items"
 446
 447     id = Column(Integer, primary_key=True)
 448     media_entry = Column(
 449         Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
 450     collection = Column(Integer, ForeignKey(Collection.id), nullable=False)
 451     note = Column(UnicodeText, nullable=True)
 452     added = Column(DateTime, nullable=False, default=datetime.datetime.now)
 453     position = Column(Integer)
 454
 455     # Cascade: CollectionItems are owned by their Collection. So do the full thing.
 456     in_collection = relationship(Collection,
 457                                  backref=backref(
 458                                      "collection_items",
 459                                      cascade="all, delete-orphan"))
 460
 461     get_media_entry = relationship(MediaEntry)
 462
 463     __table_args__ = (
 464         UniqueConstraint('collection', 'media_entry'),
 465         {})
 466
 467     @property
 468     def dict_view(self):
 469         """A dict like view on this object"""
 470         return DictReadAttrProxy(self)
 471
 472
 473 class ProcessingMetaData(Base):
 474     __tablename__ = 'core__processing_metadata'
 475
 476     id = Column(Integer, primary_key=True)
 477     media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False,
 478             index=True)
 479     media_entry = relationship(MediaEntry,
 480             backref=backref('processing_metadata',
 481                 cascade='all, delete-orphan'))
 482     callback_url = Column(Unicode)
 483
 484     @property
 485     def dict_view(self):
 486         """A dict like view on this object"""
 487         return DictReadAttrProxy(self)
 488
 489
 490 class CommentSubscription(Base):
 491     __tablename__ = 'core__comment_subscriptions'
 492     id = Column(Integer, primary_key=True)
 493
 494     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 495
 496     media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False)
 497     media_entry = relationship(MediaEntry,
 498                         backref=backref('comment_subscriptions',
 499                                         cascade='all, delete-orphan'))
 500
 501     user_id = Column(Integer, ForeignKey(User.id), nullable=False)
 502     user = relationship(User,
 503                         backref=backref('comment_subscriptions',
 504                                         cascade='all, delete-orphan'))
 505
 506     notify = Column(Boolean, nullable=False, default=True)
 507     send_email = Column(Boolean, nullable=False, default=True)
 508
 509     def __repr__(self):
 510         return ('<{classname} #{id}: {user} {media} notify: '
 511                 '{notify} email: {email}>').format(
 512             id=self.id,
 513             classname=self.__class__.__name__,
 514             user=self.user,
 515             media=self.media_entry,
 516             notify=self.notify,
 517             email=self.send_email)
 518
 519
 520 class Notification(Base):
 521     __tablename__ = 'core__notifications'
 522     id = Column(Integer, primary_key=True)
 523     type = Column(Unicode)
 524
 525     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 526
 527     user_id = Column(Integer, ForeignKey('core__users.id'), nullable=False,
 528                      index=True)
 529     seen = Column(Boolean, default=lambda: False, index=True)
 530     user = relationship(
 531         User,
 532         backref=backref('notifications', cascade='all, delete-orphan'))
 533
 534     __mapper_args__ = {
 535         'polymorphic_identity': 'notification',
 536         'polymorphic_on': type
 537     }
 538
 539     def __repr__(self):
 540         return '<{klass} #{id}: {user}: {subject} ({seen})>'.format(
 541             id=self.id,
 542             klass=self.__class__.__name__,
 543             user=self.user,
 544             subject=getattr(self, 'subject', None),
 545             seen='unseen' if not self.seen else 'seen')
 546
 547
 548 class CommentNotification(Notification):
 549     __tablename__ = 'core__comment_notifications'
 550     id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
 551
 552     subject_id = Column(Integer, ForeignKey(MediaComment.id))
 553     subject = relationship(
 554         MediaComment,
 555         backref=backref('comment_notifications', cascade='all, delete-orphan'))
 556
 557     __mapper_args__ = {
 558         'polymorphic_identity': 'comment_notification'
 559     }
 560
 561
 562 class ProcessingNotification(Notification):
 563     __tablename__ = 'core__processing_notifications'
 564
 565     id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
 566
 567     subject_id = Column(Integer, ForeignKey(MediaEntry.id))
 568     subject = relationship(
 569         MediaEntry,
 570         backref=backref('processing_notifications',
 571                         cascade='all, delete-orphan'))
 572
 573     __mapper_args__ = {
 574         'polymorphic_identity': 'processing_notification'
 575     }
 576
 577
 578 with_polymorphic(
 579     Notification,
 580     [ProcessingNotification, CommentNotification])
 581
 582 MODELS = [
 583     User, MediaEntry, Tag, MediaTag, MediaComment, Collection, CollectionItem,
 584     MediaFile, FileKeynames, MediaAttachmentFile, ProcessingMetaData,
 585     Notification, CommentNotification, ProcessingNotification,
 586     CommentSubscription]
 587
 588
 589 ######################################################
 590 # Special, migrations-tracking table
 591 #
 592 # Not listed in MODELS because this is special and not
 593 # really migrated, but used for migrations (for now)
 594 ######################################################
 595
 596 class MigrationData(Base):
 597     __tablename__ = "core__migrations"
 598
 599     name = Column(Unicode, primary_key=True)
 600     version = Column(Integer, nullable=False, default=0)
 601
 602 ######################################################
 603
 604
 605 def show_table_init(engine_uri):
 606     if engine_uri is None:
 607         engine_uri = 'sqlite:///:memory:'
 608     from sqlalchemy import create_engine
 609     engine = create_engine(engine_uri, echo=True)
 610
 611     Base.metadata.create_all(engine)
 612
 613
 614 if __name__ == '__main__':
 615     from sys import argv
 616     print repr(argv)
 617     if len(argv) == 2:
 618         uri = argv[1]
 619     else:
 620         uri = None
 621     show_table_init(uri)