Merge remote-tracking branch 'refs/remotes/tsyesika/master'
[mediagoblin.git] / mediagoblin / db / models.py
1 # GNU MediaGoblin -- federated, autonomous media hosting
2 # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
17 """
18 TODO: indexes on foreignkeys, where useful.
19 """
20
21 import logging
22 import datetime
23
24 from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \
25 Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \
26 SmallInteger
27 from sqlalchemy.orm import relationship, backref, with_polymorphic
28 from sqlalchemy.orm.collections import attribute_mapped_collection
29 from sqlalchemy.sql.expression import desc
30 from sqlalchemy.ext.associationproxy import association_proxy
31 from sqlalchemy.util import memoized_property
32
33
34 from mediagoblin.db.extratypes import PathTupleWithSlashes, JSONEncoded
35 from mediagoblin.db.base import Base, DictReadAttrProxy
36 from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, \
37 MediaCommentMixin, CollectionMixin, CollectionItemMixin
38 from mediagoblin.tools.files import delete_media_files
39 from mediagoblin.tools.common import import_component
40
41 # It's actually kind of annoying how sqlalchemy-migrate does this, if
42 # I understand it right, but whatever. Anyway, don't remove this :P
43 #
44 # We could do migration calls more manually instead of relying on
45 # this import-based meddling...
46 from migrate import changeset
47
48 _log = logging.getLogger(__name__)
49
50
51 class User(Base, UserMixin):
52 """
53 TODO: We should consider moving some rarely used fields
54 into some sort of "shadow" table.
55 """
56 __tablename__ = "core__users"
57
58 id = Column(Integer, primary_key=True)
59 username = Column(Unicode, nullable=False, unique=True)
60 # Note: no db uniqueness constraint on email because it's not
61 # reliable (many email systems case insensitive despite against
62 # the RFC) and because it would be a mess to implement at this
63 # point.
64 email = Column(Unicode, nullable=False)
65 pw_hash = Column(Unicode)
66 email_verified = Column(Boolean, default=False)
67 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
68 status = Column(Unicode, default=u"needs_email_verification", nullable=False)
69 # Intented to be nullable=False, but migrations would not work for it
70 # set to nullable=True implicitly.
71 wants_comment_notification = Column(Boolean, default=True)
72 license_preference = Column(Unicode)
73 is_admin = Column(Boolean, default=False, nullable=False)
74 url = Column(Unicode)
75 bio = Column(UnicodeText) # ??
76
77 ## TODO
78 # plugin data would be in a separate model
79
80 def __repr__(self):
81 return '<{0} #{1} {2} {3} "{4}">'.format(
82 self.__class__.__name__,
83 self.id,
84 'verified' if self.email_verified else 'non-verified',
85 'admin' if self.is_admin else 'user',
86 self.username)
87
88 def delete(self, **kwargs):
89 """Deletes a User and all related entries/comments/files/..."""
90 # Collections get deleted by relationships.
91
92 media_entries = MediaEntry.query.filter(MediaEntry.uploader == self.id)
93 for media in media_entries:
94 # TODO: Make sure that "MediaEntry.delete()" also deletes
95 # all related files/Comments
96 media.delete(del_orphan_tags=False, commit=False)
97
98 # Delete now unused tags
99 # TODO: import here due to cyclic imports!!! This cries for refactoring
100 from mediagoblin.db.util import clean_orphan_tags
101 clean_orphan_tags(commit=False)
102
103 # Delete user, pass through commit=False/True in kwargs
104 super(User, self).delete(**kwargs)
105 _log.info('Deleted user "{0}" account'.format(self.username))
106
107
108 class Client(Base):
109 """
110 Model representing a client - Used for API Auth
111 """
112 __tablename__ = "core__clients"
113
114 id = Column(Unicode, nullable=True, primary_key=True)
115 secret = Column(Unicode, nullable=False)
116 expirey = Column(DateTime, nullable=True)
117 application_type = Column(Unicode, nullable=False)
118 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
119 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
120
121 # optional stuff
122 redirect_uri = Column(JSONEncoded, nullable=True)
123 logo_url = Column(Unicode, nullable=True)
124 application_name = Column(Unicode, nullable=True)
125 contacts = Column(JSONEncoded, nullable=True)
126
127 def __repr__(self):
128 if self.application_name:
129 return "<Client {0} - {1}>".format(self.application_name, self.id)
130 else:
131 return "<Client {0}>".format(self.id)
132
133 class RequestToken(Base):
134 """
135 Model for representing the request tokens
136 """
137 __tablename__ = "core__request_tokens"
138
139 token = Column(Unicode, primary_key=True)
140 secret = Column(Unicode, nullable=False)
141 client = Column(Unicode, ForeignKey(Client.id))
142 user = Column(Integer, ForeignKey(User.id), nullable=True)
143 used = Column(Boolean, default=False)
144 authenticated = Column(Boolean, default=False)
145 verifier = Column(Unicode, nullable=True)
146 callback = Column(Unicode, nullable=False, default=u"oob")
147 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
148 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
149
150 class AccessToken(Base):
151 """
152 Model for representing the access tokens
153 """
154 __tablename__ = "core__access_tokens"
155
156 token = Column(Unicode, nullable=False, primary_key=True)
157 secret = Column(Unicode, nullable=False)
158 user = Column(Integer, ForeignKey(User.id))
159 request_token = Column(Unicode, ForeignKey(RequestToken.token))
160 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
161 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
162
163
164 class NonceTimestamp(Base):
165 """
166 A place the timestamp and nonce can be stored - this is for OAuth1
167 """
168 __tablename__ = "core__nonce_timestamps"
169
170 nonce = Column(Unicode, nullable=False, primary_key=True)
171 timestamp = Column(DateTime, nullable=False, primary_key=True)
172
173
174 class MediaEntry(Base, MediaEntryMixin):
175 """
176 TODO: Consider fetching the media_files using join
177 """
178 __tablename__ = "core__media_entries"
179
180 id = Column(Integer, primary_key=True)
181 uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
182 title = Column(Unicode, nullable=False)
183 slug = Column(Unicode)
184 created = Column(DateTime, nullable=False, default=datetime.datetime.now,
185 index=True)
186 description = Column(UnicodeText) # ??
187 media_type = Column(Unicode, nullable=False)
188 state = Column(Unicode, default=u'unprocessed', nullable=False)
189 # or use sqlalchemy.types.Enum?
190 license = Column(Unicode)
191 collected = Column(Integer, default=0)
192
193 fail_error = Column(Unicode)
194 fail_metadata = Column(JSONEncoded)
195
196 transcoding_progress = Column(SmallInteger)
197
198 queued_media_file = Column(PathTupleWithSlashes)
199
200 queued_task_id = Column(Unicode)
201
202 __table_args__ = (
203 UniqueConstraint('uploader', 'slug'),
204 {})
205
206 get_uploader = relationship(User)
207
208 media_files_helper = relationship("MediaFile",
209 collection_class=attribute_mapped_collection("name"),
210 cascade="all, delete-orphan"
211 )
212 media_files = association_proxy('media_files_helper', 'file_path',
213 creator=lambda k, v: MediaFile(name=k, file_path=v)
214 )
215
216 attachment_files_helper = relationship("MediaAttachmentFile",
217 cascade="all, delete-orphan",
218 order_by="MediaAttachmentFile.created"
219 )
220 attachment_files = association_proxy("attachment_files_helper", "dict_view",
221 creator=lambda v: MediaAttachmentFile(
222 name=v["name"], filepath=v["filepath"])
223 )
224
225 tags_helper = relationship("MediaTag",
226 cascade="all, delete-orphan" # should be automatically deleted
227 )
228 tags = association_proxy("tags_helper", "dict_view",
229 creator=lambda v: MediaTag(name=v["name"], slug=v["slug"])
230 )
231
232 collections_helper = relationship("CollectionItem",
233 cascade="all, delete-orphan"
234 )
235 collections = association_proxy("collections_helper", "in_collection")
236
237 ## TODO
238 # fail_error
239
240 def get_comments(self, ascending=False):
241 order_col = MediaComment.created
242 if not ascending:
243 order_col = desc(order_col)
244 return self.all_comments.order_by(order_col)
245
246 def url_to_prev(self, urlgen):
247 """get the next 'newer' entry by this user"""
248 media = MediaEntry.query.filter(
249 (MediaEntry.uploader == self.uploader)
250 & (MediaEntry.state == u'processed')
251 & (MediaEntry.id > self.id)).order_by(MediaEntry.id).first()
252
253 if media is not None:
254 return media.url_for_self(urlgen)
255
256 def url_to_next(self, urlgen):
257 """get the next 'older' entry by this user"""
258 media = MediaEntry.query.filter(
259 (MediaEntry.uploader == self.uploader)
260 & (MediaEntry.state == u'processed')
261 & (MediaEntry.id < self.id)).order_by(desc(MediaEntry.id)).first()
262
263 if media is not None:
264 return media.url_for_self(urlgen)
265
266 @property
267 def media_data(self):
268 return getattr(self, self.media_data_ref)
269
270 def media_data_init(self, **kwargs):
271 """
272 Initialize or update the contents of a media entry's media_data row
273 """
274 media_data = self.media_data
275
276 if media_data is None:
277 # Get the correct table:
278 table = import_component(self.media_type + '.models:DATA_MODEL')
279 # No media data, so actually add a new one
280 media_data = table(**kwargs)
281 # Get the relationship set up.
282 media_data.get_media_entry = self
283 else:
284 # Update old media data
285 for field, value in kwargs.iteritems():
286 setattr(media_data, field, value)
287
288 @memoized_property
289 def media_data_ref(self):
290 return import_component(self.media_type + '.models:BACKREF_NAME')
291
292 def __repr__(self):
293 safe_title = self.title.encode('ascii', 'replace')
294
295 return '<{classname} {id}: {title}>'.format(
296 classname=self.__class__.__name__,
297 id=self.id,
298 title=safe_title)
299
300 def delete(self, del_orphan_tags=True, **kwargs):
301 """Delete MediaEntry and all related files/attachments/comments
302
303 This will *not* automatically delete unused collections, which
304 can remain empty...
305
306 :param del_orphan_tags: True/false if we delete unused Tags too
307 :param commit: True/False if this should end the db transaction"""
308 # User's CollectionItems are automatically deleted via "cascade".
309 # Comments on this Media are deleted by cascade, hopefully.
310
311 # Delete all related files/attachments
312 try:
313 delete_media_files(self)
314 except OSError, error:
315 # Returns list of files we failed to delete
316 _log.error('No such files from the user "{1}" to delete: '
317 '{0}'.format(str(error), self.get_uploader))
318 _log.info('Deleted Media entry id "{0}"'.format(self.id))
319 # Related MediaTag's are automatically cleaned, but we might
320 # want to clean out unused Tag's too.
321 if del_orphan_tags:
322 # TODO: Import here due to cyclic imports!!!
323 # This cries for refactoring
324 from mediagoblin.db.util import clean_orphan_tags
325 clean_orphan_tags(commit=False)
326 # pass through commit=False/True in kwargs
327 super(MediaEntry, self).delete(**kwargs)
328
329
330 class FileKeynames(Base):
331 """
332 keywords for various places.
333 currently the MediaFile keys
334 """
335 __tablename__ = "core__file_keynames"
336 id = Column(Integer, primary_key=True)
337 name = Column(Unicode, unique=True)
338
339 def __repr__(self):
340 return "<FileKeyname %r: %r>" % (self.id, self.name)
341
342 @classmethod
343 def find_or_new(cls, name):
344 t = cls.query.filter_by(name=name).first()
345 if t is not None:
346 return t
347 return cls(name=name)
348
349
350 class MediaFile(Base):
351 """
352 TODO: Highly consider moving "name" into a new table.
353 TODO: Consider preloading said table in software
354 """
355 __tablename__ = "core__mediafiles"
356
357 media_entry = Column(
358 Integer, ForeignKey(MediaEntry.id),
359 nullable=False)
360 name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False)
361 file_path = Column(PathTupleWithSlashes)
362
363 __table_args__ = (
364 PrimaryKeyConstraint('media_entry', 'name_id'),
365 {})
366
367 def __repr__(self):
368 return "<MediaFile %s: %r>" % (self.name, self.file_path)
369
370 name_helper = relationship(FileKeynames, lazy="joined", innerjoin=True)
371 name = association_proxy('name_helper', 'name',
372 creator=FileKeynames.find_or_new
373 )
374
375
376 class MediaAttachmentFile(Base):
377 __tablename__ = "core__attachment_files"
378
379 id = Column(Integer, primary_key=True)
380 media_entry = Column(
381 Integer, ForeignKey(MediaEntry.id),
382 nullable=False)
383 name = Column(Unicode, nullable=False)
384 filepath = Column(PathTupleWithSlashes)
385 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
386
387 @property
388 def dict_view(self):
389 """A dict like view on this object"""
390 return DictReadAttrProxy(self)
391
392
393 class Tag(Base):
394 __tablename__ = "core__tags"
395
396 id = Column(Integer, primary_key=True)
397 slug = Column(Unicode, nullable=False, unique=True)
398
399 def __repr__(self):
400 return "<Tag %r: %r>" % (self.id, self.slug)
401
402 @classmethod
403 def find_or_new(cls, slug):
404 t = cls.query.filter_by(slug=slug).first()
405 if t is not None:
406 return t
407 return cls(slug=slug)
408
409
410 class MediaTag(Base):
411 __tablename__ = "core__media_tags"
412
413 id = Column(Integer, primary_key=True)
414 media_entry = Column(
415 Integer, ForeignKey(MediaEntry.id),
416 nullable=False, index=True)
417 tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True)
418 name = Column(Unicode)
419 # created = Column(DateTime, nullable=False, default=datetime.datetime.now)
420
421 __table_args__ = (
422 UniqueConstraint('tag', 'media_entry'),
423 {})
424
425 tag_helper = relationship(Tag)
426 slug = association_proxy('tag_helper', 'slug',
427 creator=Tag.find_or_new
428 )
429
430 def __init__(self, name=None, slug=None):
431 Base.__init__(self)
432 if name is not None:
433 self.name = name
434 if slug is not None:
435 self.tag_helper = Tag.find_or_new(slug)
436
437 @property
438 def dict_view(self):
439 """A dict like view on this object"""
440 return DictReadAttrProxy(self)
441
442
443 class MediaComment(Base, MediaCommentMixin):
444 __tablename__ = "core__media_comments"
445
446 id = Column(Integer, primary_key=True)
447 media_entry = Column(
448 Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
449 author = Column(Integer, ForeignKey(User.id), nullable=False)
450 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
451 content = Column(UnicodeText, nullable=False)
452
453 # Cascade: Comments are owned by their creator. So do the full thing.
454 # lazy=dynamic: People might post a *lot* of comments,
455 # so make the "posted_comments" a query-like thing.
456 get_author = relationship(User,
457 backref=backref("posted_comments",
458 lazy="dynamic",
459 cascade="all, delete-orphan"))
460 get_entry = relationship(MediaEntry,
461 backref=backref("comments",
462 lazy="dynamic",
463 cascade="all, delete-orphan"))
464
465 # Cascade: Comments are somewhat owned by their MediaEntry.
466 # So do the full thing.
467 # lazy=dynamic: MediaEntries might have many comments,
468 # so make the "all_comments" a query-like thing.
469 get_media_entry = relationship(MediaEntry,
470 backref=backref("all_comments",
471 lazy="dynamic",
472 cascade="all, delete-orphan"))
473
474
475 class Collection(Base, CollectionMixin):
476 """An 'album' or 'set' of media by a user.
477
478 On deletion, contained CollectionItems get automatically reaped via
479 SQL cascade"""
480 __tablename__ = "core__collections"
481
482 id = Column(Integer, primary_key=True)
483 title = Column(Unicode, nullable=False)
484 slug = Column(Unicode)
485 created = Column(DateTime, nullable=False, default=datetime.datetime.now,
486 index=True)
487 description = Column(UnicodeText)
488 creator = Column(Integer, ForeignKey(User.id), nullable=False)
489 # TODO: No of items in Collection. Badly named, can we migrate to num_items?
490 items = Column(Integer, default=0)
491
492 # Cascade: Collections are owned by their creator. So do the full thing.
493 get_creator = relationship(User,
494 backref=backref("collections",
495 cascade="all, delete-orphan"))
496
497 __table_args__ = (
498 UniqueConstraint('creator', 'slug'),
499 {})
500
501 def get_collection_items(self, ascending=False):
502 #TODO, is this still needed with self.collection_items being available?
503 order_col = CollectionItem.position
504 if not ascending:
505 order_col = desc(order_col)
506 return CollectionItem.query.filter_by(
507 collection=self.id).order_by(order_col)
508
509
510 class CollectionItem(Base, CollectionItemMixin):
511 __tablename__ = "core__collection_items"
512
513 id = Column(Integer, primary_key=True)
514 media_entry = Column(
515 Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
516 collection = Column(Integer, ForeignKey(Collection.id), nullable=False)
517 note = Column(UnicodeText, nullable=True)
518 added = Column(DateTime, nullable=False, default=datetime.datetime.now)
519 position = Column(Integer)
520
521 # Cascade: CollectionItems are owned by their Collection. So do the full thing.
522 in_collection = relationship(Collection,
523 backref=backref(
524 "collection_items",
525 cascade="all, delete-orphan"))
526
527 get_media_entry = relationship(MediaEntry)
528
529 __table_args__ = (
530 UniqueConstraint('collection', 'media_entry'),
531 {})
532
533 @property
534 def dict_view(self):
535 """A dict like view on this object"""
536 return DictReadAttrProxy(self)
537
538
539 class ProcessingMetaData(Base):
540 __tablename__ = 'core__processing_metadata'
541
542 id = Column(Integer, primary_key=True)
543 media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False,
544 index=True)
545 media_entry = relationship(MediaEntry,
546 backref=backref('processing_metadata',
547 cascade='all, delete-orphan'))
548 callback_url = Column(Unicode)
549
550 @property
551 def dict_view(self):
552 """A dict like view on this object"""
553 return DictReadAttrProxy(self)
554
555
556 class CommentSubscription(Base):
557 __tablename__ = 'core__comment_subscriptions'
558 id = Column(Integer, primary_key=True)
559
560 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
561
562 media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False)
563 media_entry = relationship(MediaEntry,
564 backref=backref('comment_subscriptions',
565 cascade='all, delete-orphan'))
566
567 user_id = Column(Integer, ForeignKey(User.id), nullable=False)
568 user = relationship(User,
569 backref=backref('comment_subscriptions',
570 cascade='all, delete-orphan'))
571
572 notify = Column(Boolean, nullable=False, default=True)
573 send_email = Column(Boolean, nullable=False, default=True)
574
575 def __repr__(self):
576 return ('<{classname} #{id}: {user} {media} notify: '
577 '{notify} email: {email}>').format(
578 id=self.id,
579 classname=self.__class__.__name__,
580 user=self.user,
581 media=self.media_entry,
582 notify=self.notify,
583 email=self.send_email)
584
585
586 class Notification(Base):
587 __tablename__ = 'core__notifications'
588 id = Column(Integer, primary_key=True)
589 type = Column(Unicode)
590
591 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
592
593 user_id = Column(Integer, ForeignKey('core__users.id'), nullable=False,
594 index=True)
595 seen = Column(Boolean, default=lambda: False, index=True)
596 user = relationship(
597 User,
598 backref=backref('notifications', cascade='all, delete-orphan'))
599
600 __mapper_args__ = {
601 'polymorphic_identity': 'notification',
602 'polymorphic_on': type
603 }
604
605 def __repr__(self):
606 return '<{klass} #{id}: {user}: {subject} ({seen})>'.format(
607 id=self.id,
608 klass=self.__class__.__name__,
609 user=self.user,
610 subject=getattr(self, 'subject', None),
611 seen='unseen' if not self.seen else 'seen')
612
613
614 class CommentNotification(Notification):
615 __tablename__ = 'core__comment_notifications'
616 id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
617
618 subject_id = Column(Integer, ForeignKey(MediaComment.id))
619 subject = relationship(
620 MediaComment,
621 backref=backref('comment_notifications', cascade='all, delete-orphan'))
622
623 __mapper_args__ = {
624 'polymorphic_identity': 'comment_notification'
625 }
626
627
628 class ProcessingNotification(Notification):
629 __tablename__ = 'core__processing_notifications'
630
631 id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
632
633 subject_id = Column(Integer, ForeignKey(MediaEntry.id))
634 subject = relationship(
635 MediaEntry,
636 backref=backref('processing_notifications',
637 cascade='all, delete-orphan'))
638
639 __mapper_args__ = {
640 'polymorphic_identity': 'processing_notification'
641 }
642
643
644 with_polymorphic(
645 Notification,
646 [ProcessingNotification, CommentNotification])
647
648 MODELS = [
649 User, Client, RequestToken, AccessToken, NonceTimestamp, MediaEntry, Tag,
650 MediaTag, MediaComment, Collection, CollectionItem, MediaFile, FileKeynames,
651 MediaAttachmentFile, ProcessingMetaData, Notification, CommentNotification,
652 ProcessingNotification, CommentSubscription]
653
654 """
655 Foundations are the default rows that are created immediately after the tables
656 are initialized. Each entry to this dictionary should be in the format of:
657 ModelConstructorObject:List of Dictionaries
658 (Each Dictionary represents a row on the Table to be created, containing each
659 of the columns' names as a key string, and each of the columns' values as a
660 value)
661
662 ex. [NOTE THIS IS NOT BASED OFF OF OUR USER TABLE]
663 user_foundations = [{'name':u'Joanna', 'age':24},
664 {'name':u'Andrea', 'age':41}]
665
666 FOUNDATIONS = {User:user_foundations}
667 """
668 FOUNDATIONS = {}
669
670 ######################################################
671 # Special, migrations-tracking table
672 #
673 # Not listed in MODELS because this is special and not
674 # really migrated, but used for migrations (for now)
675 ######################################################
676
677 class MigrationData(Base):
678 __tablename__ = "core__migrations"
679
680 name = Column(Unicode, primary_key=True)
681 version = Column(Integer, nullable=False, default=0)
682
683 ######################################################
684
685
686 def show_table_init(engine_uri):
687 if engine_uri is None:
688 engine_uri = 'sqlite:///:memory:'
689 from sqlalchemy import create_engine
690 engine = create_engine(engine_uri, echo=True)
691
692 Base.metadata.create_all(engine)
693
694
695 if __name__ == '__main__':
696 from sys import argv
697 print repr(argv)
698 if len(argv) == 2:
699 uri = argv[1]
700 else:
701 uri = None
702 show_table_init(uri)