Write template to render blog about page.
[mediagoblin.git] / mediagoblin / db / models.py
1 # GNU MediaGoblin -- federated, autonomous media hosting
2 # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
17 """
18 TODO: indexes on foreignkeys, where useful.
19 """
20
21 import logging
22 import datetime
23
24 from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \
25 Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \
26 SmallInteger
27 from sqlalchemy.orm import relationship, backref, with_polymorphic
28 from sqlalchemy.orm.collections import attribute_mapped_collection
29 from sqlalchemy.sql.expression import desc
30 from sqlalchemy.ext.associationproxy import association_proxy
31 from sqlalchemy.util import memoized_property
32
33
34 from mediagoblin.db.extratypes import PathTupleWithSlashes, JSONEncoded
35 from mediagoblin.db.base import Base, DictReadAttrProxy
36 from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, \
37 MediaCommentMixin, CollectionMixin, CollectionItemMixin
38 from mediagoblin.tools.files import delete_media_files
39 from mediagoblin.tools.common import import_component
40
41 # It's actually kind of annoying how sqlalchemy-migrate does this, if
42 # I understand it right, but whatever. Anyway, don't remove this :P
43 #
44 # We could do migration calls more manually instead of relying on
45 # this import-based meddling...
46 from migrate import changeset
47
48 _log = logging.getLogger(__name__)
49
50
51 class User(Base, UserMixin):
52 """
53 TODO: We should consider moving some rarely used fields
54 into some sort of "shadow" table.
55 """
56 __tablename__ = "core__users"
57
58 id = Column(Integer, primary_key=True)
59 username = Column(Unicode, nullable=False, unique=True)
60 # Note: no db uniqueness constraint on email because it's not
61 # reliable (many email systems case insensitive despite against
62 # the RFC) and because it would be a mess to implement at this
63 # point.
64 email = Column(Unicode, nullable=False)
65 pw_hash = Column(Unicode)
66 email_verified = Column(Boolean, default=False)
67 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
68 status = Column(Unicode, default=u"needs_email_verification", nullable=False)
69 # Intented to be nullable=False, but migrations would not work for it
70 # set to nullable=True implicitly.
71 wants_comment_notification = Column(Boolean, default=True)
72 wants_notifications = Column(Boolean, default=True)
73 license_preference = Column(Unicode)
74 is_admin = Column(Boolean, default=False, nullable=False)
75 url = Column(Unicode)
76 bio = Column(UnicodeText) # ??
77
78 ## TODO
79 # plugin data would be in a separate model
80
81 def __repr__(self):
82 return '<{0} #{1} {2} {3} "{4}">'.format(
83 self.__class__.__name__,
84 self.id,
85 'verified' if self.email_verified else 'non-verified',
86 'admin' if self.is_admin else 'user',
87 self.username)
88
89 def delete(self, **kwargs):
90 """Deletes a User and all related entries/comments/files/..."""
91 # Collections get deleted by relationships.
92
93 media_entries = MediaEntry.query.filter(MediaEntry.uploader == self.id)
94 for media in media_entries:
95 # TODO: Make sure that "MediaEntry.delete()" also deletes
96 # all related files/Comments
97 media.delete(del_orphan_tags=False, commit=False)
98
99 # Delete now unused tags
100 # TODO: import here due to cyclic imports!!! This cries for refactoring
101 from mediagoblin.db.util import clean_orphan_tags
102 clean_orphan_tags(commit=False)
103
104 # Delete user, pass through commit=False/True in kwargs
105 super(User, self).delete(**kwargs)
106 _log.info('Deleted user "{0}" account'.format(self.username))
107
108
109 class Client(Base):
110 """
111 Model representing a client - Used for API Auth
112 """
113 __tablename__ = "core__clients"
114
115 id = Column(Unicode, nullable=True, primary_key=True)
116 secret = Column(Unicode, nullable=False)
117 expirey = Column(DateTime, nullable=True)
118 application_type = Column(Unicode, nullable=False)
119 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
120 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
121
122 # optional stuff
123 redirect_uri = Column(JSONEncoded, nullable=True)
124 logo_url = Column(Unicode, nullable=True)
125 application_name = Column(Unicode, nullable=True)
126 contacts = Column(JSONEncoded, nullable=True)
127
128 def __repr__(self):
129 if self.application_name:
130 return "<Client {0} - {1}>".format(self.application_name, self.id)
131 else:
132 return "<Client {0}>".format(self.id)
133
134 class RequestToken(Base):
135 """
136 Model for representing the request tokens
137 """
138 __tablename__ = "core__request_tokens"
139
140 token = Column(Unicode, primary_key=True)
141 secret = Column(Unicode, nullable=False)
142 client = Column(Unicode, ForeignKey(Client.id))
143 user = Column(Integer, ForeignKey(User.id), nullable=True)
144 used = Column(Boolean, default=False)
145 authenticated = Column(Boolean, default=False)
146 verifier = Column(Unicode, nullable=True)
147 callback = Column(Unicode, nullable=False, default=u"oob")
148 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
149 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
150
151 class AccessToken(Base):
152 """
153 Model for representing the access tokens
154 """
155 __tablename__ = "core__access_tokens"
156
157 token = Column(Unicode, nullable=False, primary_key=True)
158 secret = Column(Unicode, nullable=False)
159 user = Column(Integer, ForeignKey(User.id))
160 request_token = Column(Unicode, ForeignKey(RequestToken.token))
161 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
162 updated = Column(DateTime, nullable=False, default=datetime.datetime.now)
163
164
165 class NonceTimestamp(Base):
166 """
167 A place the timestamp and nonce can be stored - this is for OAuth1
168 """
169 __tablename__ = "core__nonce_timestamps"
170
171 nonce = Column(Unicode, nullable=False, primary_key=True)
172 timestamp = Column(DateTime, nullable=False, primary_key=True)
173
174
175 class MediaEntry(Base, MediaEntryMixin):
176 """
177 TODO: Consider fetching the media_files using join
178 """
179 __tablename__ = "core__media_entries"
180
181 id = Column(Integer, primary_key=True)
182 uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
183 title = Column(Unicode, nullable=False)
184 slug = Column(Unicode)
185 created = Column(DateTime, nullable=False, default=datetime.datetime.now,
186 index=True)
187 description = Column(UnicodeText) # ??
188 media_type = Column(Unicode, nullable=False)
189 state = Column(Unicode, default=u'unprocessed', nullable=False)
190 # or use sqlalchemy.types.Enum?
191 license = Column(Unicode)
192 collected = Column(Integer, default=0)
193
194 fail_error = Column(Unicode)
195 fail_metadata = Column(JSONEncoded)
196
197 transcoding_progress = Column(SmallInteger)
198
199 queued_media_file = Column(PathTupleWithSlashes)
200
201 queued_task_id = Column(Unicode)
202
203 __table_args__ = (
204 UniqueConstraint('uploader', 'slug'),
205 {})
206
207 get_uploader = relationship(User)
208
209 media_files_helper = relationship("MediaFile",
210 collection_class=attribute_mapped_collection("name"),
211 cascade="all, delete-orphan"
212 )
213 media_files = association_proxy('media_files_helper', 'file_path',
214 creator=lambda k, v: MediaFile(name=k, file_path=v)
215 )
216
217 attachment_files_helper = relationship("MediaAttachmentFile",
218 cascade="all, delete-orphan",
219 order_by="MediaAttachmentFile.created"
220 )
221 attachment_files = association_proxy("attachment_files_helper", "dict_view",
222 creator=lambda v: MediaAttachmentFile(
223 name=v["name"], filepath=v["filepath"])
224 )
225
226 tags_helper = relationship("MediaTag",
227 cascade="all, delete-orphan" # should be automatically deleted
228 )
229 tags = association_proxy("tags_helper", "dict_view",
230 creator=lambda v: MediaTag(name=v["name"], slug=v["slug"])
231 )
232
233 collections_helper = relationship("CollectionItem",
234 cascade="all, delete-orphan"
235 )
236 collections = association_proxy("collections_helper", "in_collection")
237
238 ## TODO
239 # fail_error
240
241 def get_comments(self, ascending=False):
242 order_col = MediaComment.created
243 if not ascending:
244 order_col = desc(order_col)
245 return self.all_comments.order_by(order_col)
246
247 def url_to_prev(self, urlgen):
248 """get the next 'newer' entry by this user"""
249 media = MediaEntry.query.filter(
250 (MediaEntry.uploader == self.uploader)
251 & (MediaEntry.state == u'processed')
252 & (MediaEntry.id > self.id)).order_by(MediaEntry.id).first()
253
254 if media is not None:
255 return media.url_for_self(urlgen)
256
257 def url_to_next(self, urlgen):
258 """get the next 'older' entry by this user"""
259 media = MediaEntry.query.filter(
260 (MediaEntry.uploader == self.uploader)
261 & (MediaEntry.state == u'processed')
262 & (MediaEntry.id < self.id)).order_by(desc(MediaEntry.id)).first()
263
264 if media is not None:
265 return media.url_for_self(urlgen)
266
267 @property
268 def media_data(self):
269 return getattr(self, self.media_data_ref)
270
271 def media_data_init(self, **kwargs):
272 """
273 Initialize or update the contents of a media entry's media_data row
274 """
275 media_data = self.media_data
276
277 if media_data is None:
278 # Get the correct table:
279 table = import_component(self.media_type + '.models:DATA_MODEL')
280 # No media data, so actually add a new one
281 media_data = table(**kwargs)
282 # Get the relationship set up.
283 media_data.get_media_entry = self
284 else:
285 # Update old media data
286 for field, value in kwargs.iteritems():
287 setattr(media_data, field, value)
288
289 @memoized_property
290 def media_data_ref(self):
291 return import_component(self.media_type + '.models:BACKREF_NAME')
292
293 def __repr__(self):
294 safe_title = self.title.encode('ascii', 'replace')
295
296 return '<{classname} {id}: {title}>'.format(
297 classname=self.__class__.__name__,
298 id=self.id,
299 title=safe_title)
300
301 def delete(self, del_orphan_tags=True, **kwargs):
302 """Delete MediaEntry and all related files/attachments/comments
303
304 This will *not* automatically delete unused collections, which
305 can remain empty...
306
307 :param del_orphan_tags: True/false if we delete unused Tags too
308 :param commit: True/False if this should end the db transaction"""
309 # User's CollectionItems are automatically deleted via "cascade".
310 # Comments on this Media are deleted by cascade, hopefully.
311
312 # Delete all related files/attachments
313 try:
314 delete_media_files(self)
315 except OSError, error:
316 # Returns list of files we failed to delete
317 _log.error('No such files from the user "{1}" to delete: '
318 '{0}'.format(str(error), self.get_uploader))
319 _log.info('Deleted Media entry id "{0}"'.format(self.id))
320 # Related MediaTag's are automatically cleaned, but we might
321 # want to clean out unused Tag's too.
322 if del_orphan_tags:
323 # TODO: Import here due to cyclic imports!!!
324 # This cries for refactoring
325 from mediagoblin.db.util import clean_orphan_tags
326 clean_orphan_tags(commit=False)
327 # pass through commit=False/True in kwargs
328 super(MediaEntry, self).delete(**kwargs)
329
330
331 class FileKeynames(Base):
332 """
333 keywords for various places.
334 currently the MediaFile keys
335 """
336 __tablename__ = "core__file_keynames"
337 id = Column(Integer, primary_key=True)
338 name = Column(Unicode, unique=True)
339
340 def __repr__(self):
341 return "<FileKeyname %r: %r>" % (self.id, self.name)
342
343 @classmethod
344 def find_or_new(cls, name):
345 t = cls.query.filter_by(name=name).first()
346 if t is not None:
347 return t
348 return cls(name=name)
349
350
351 class MediaFile(Base):
352 """
353 TODO: Highly consider moving "name" into a new table.
354 TODO: Consider preloading said table in software
355 """
356 __tablename__ = "core__mediafiles"
357
358 media_entry = Column(
359 Integer, ForeignKey(MediaEntry.id),
360 nullable=False)
361 name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False)
362 file_path = Column(PathTupleWithSlashes)
363
364 __table_args__ = (
365 PrimaryKeyConstraint('media_entry', 'name_id'),
366 {})
367
368 def __repr__(self):
369 return "<MediaFile %s: %r>" % (self.name, self.file_path)
370
371 name_helper = relationship(FileKeynames, lazy="joined", innerjoin=True)
372 name = association_proxy('name_helper', 'name',
373 creator=FileKeynames.find_or_new
374 )
375
376
377 class MediaAttachmentFile(Base):
378 __tablename__ = "core__attachment_files"
379
380 id = Column(Integer, primary_key=True)
381 media_entry = Column(
382 Integer, ForeignKey(MediaEntry.id),
383 nullable=False)
384 name = Column(Unicode, nullable=False)
385 filepath = Column(PathTupleWithSlashes)
386 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
387
388 @property
389 def dict_view(self):
390 """A dict like view on this object"""
391 return DictReadAttrProxy(self)
392
393
394 class Tag(Base):
395 __tablename__ = "core__tags"
396
397 id = Column(Integer, primary_key=True)
398 slug = Column(Unicode, nullable=False, unique=True)
399
400 def __repr__(self):
401 return "<Tag %r: %r>" % (self.id, self.slug)
402
403 @classmethod
404 def find_or_new(cls, slug):
405 t = cls.query.filter_by(slug=slug).first()
406 if t is not None:
407 return t
408 return cls(slug=slug)
409
410
411 class MediaTag(Base):
412 __tablename__ = "core__media_tags"
413
414 id = Column(Integer, primary_key=True)
415 media_entry = Column(
416 Integer, ForeignKey(MediaEntry.id),
417 nullable=False, index=True)
418 tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True)
419 name = Column(Unicode)
420 # created = Column(DateTime, nullable=False, default=datetime.datetime.now)
421
422 __table_args__ = (
423 UniqueConstraint('tag', 'media_entry'),
424 {})
425
426 tag_helper = relationship(Tag)
427 slug = association_proxy('tag_helper', 'slug',
428 creator=Tag.find_or_new
429 )
430
431 def __init__(self, name=None, slug=None):
432 Base.__init__(self)
433 if name is not None:
434 self.name = name
435 if slug is not None:
436 self.tag_helper = Tag.find_or_new(slug)
437
438 @property
439 def dict_view(self):
440 """A dict like view on this object"""
441 return DictReadAttrProxy(self)
442
443
444 class MediaComment(Base, MediaCommentMixin):
445 __tablename__ = "core__media_comments"
446
447 id = Column(Integer, primary_key=True)
448 media_entry = Column(
449 Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
450 author = Column(Integer, ForeignKey(User.id), nullable=False)
451 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
452 content = Column(UnicodeText, nullable=False)
453
454 # Cascade: Comments are owned by their creator. So do the full thing.
455 # lazy=dynamic: People might post a *lot* of comments,
456 # so make the "posted_comments" a query-like thing.
457 get_author = relationship(User,
458 backref=backref("posted_comments",
459 lazy="dynamic",
460 cascade="all, delete-orphan"))
461 get_entry = relationship(MediaEntry,
462 backref=backref("comments",
463 lazy="dynamic",
464 cascade="all, delete-orphan"))
465
466 # Cascade: Comments are somewhat owned by their MediaEntry.
467 # So do the full thing.
468 # lazy=dynamic: MediaEntries might have many comments,
469 # so make the "all_comments" a query-like thing.
470 get_media_entry = relationship(MediaEntry,
471 backref=backref("all_comments",
472 lazy="dynamic",
473 cascade="all, delete-orphan"))
474
475
476 class Collection(Base, CollectionMixin):
477 """An 'album' or 'set' of media by a user.
478
479 On deletion, contained CollectionItems get automatically reaped via
480 SQL cascade"""
481 __tablename__ = "core__collections"
482
483 id = Column(Integer, primary_key=True)
484 title = Column(Unicode, nullable=False)
485 slug = Column(Unicode)
486 created = Column(DateTime, nullable=False, default=datetime.datetime.now,
487 index=True)
488 description = Column(UnicodeText)
489 creator = Column(Integer, ForeignKey(User.id), nullable=False)
490 # TODO: No of items in Collection. Badly named, can we migrate to num_items?
491 items = Column(Integer, default=0)
492
493 # Cascade: Collections are owned by their creator. So do the full thing.
494 get_creator = relationship(User,
495 backref=backref("collections",
496 cascade="all, delete-orphan"))
497
498 __table_args__ = (
499 UniqueConstraint('creator', 'slug'),
500 {})
501
502 def get_collection_items(self, ascending=False):
503 #TODO, is this still needed with self.collection_items being available?
504 order_col = CollectionItem.position
505 if not ascending:
506 order_col = desc(order_col)
507 return CollectionItem.query.filter_by(
508 collection=self.id).order_by(order_col)
509
510
511 class CollectionItem(Base, CollectionItemMixin):
512 __tablename__ = "core__collection_items"
513
514 id = Column(Integer, primary_key=True)
515 media_entry = Column(
516 Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
517 collection = Column(Integer, ForeignKey(Collection.id), nullable=False)
518 note = Column(UnicodeText, nullable=True)
519 added = Column(DateTime, nullable=False, default=datetime.datetime.now)
520 position = Column(Integer)
521
522 # Cascade: CollectionItems are owned by their Collection. So do the full thing.
523 in_collection = relationship(Collection,
524 backref=backref(
525 "collection_items",
526 cascade="all, delete-orphan"))
527
528 get_media_entry = relationship(MediaEntry)
529
530 __table_args__ = (
531 UniqueConstraint('collection', 'media_entry'),
532 {})
533
534 @property
535 def dict_view(self):
536 """A dict like view on this object"""
537 return DictReadAttrProxy(self)
538
539
540 class ProcessingMetaData(Base):
541 __tablename__ = 'core__processing_metadata'
542
543 id = Column(Integer, primary_key=True)
544 media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False,
545 index=True)
546 media_entry = relationship(MediaEntry,
547 backref=backref('processing_metadata',
548 cascade='all, delete-orphan'))
549 callback_url = Column(Unicode)
550
551 @property
552 def dict_view(self):
553 """A dict like view on this object"""
554 return DictReadAttrProxy(self)
555
556
557 class CommentSubscription(Base):
558 __tablename__ = 'core__comment_subscriptions'
559 id = Column(Integer, primary_key=True)
560
561 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
562
563 media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False)
564 media_entry = relationship(MediaEntry,
565 backref=backref('comment_subscriptions',
566 cascade='all, delete-orphan'))
567
568 user_id = Column(Integer, ForeignKey(User.id), nullable=False)
569 user = relationship(User,
570 backref=backref('comment_subscriptions',
571 cascade='all, delete-orphan'))
572
573 notify = Column(Boolean, nullable=False, default=True)
574 send_email = Column(Boolean, nullable=False, default=True)
575
576 def __repr__(self):
577 return ('<{classname} #{id}: {user} {media} notify: '
578 '{notify} email: {email}>').format(
579 id=self.id,
580 classname=self.__class__.__name__,
581 user=self.user,
582 media=self.media_entry,
583 notify=self.notify,
584 email=self.send_email)
585
586
587 class Notification(Base):
588 __tablename__ = 'core__notifications'
589 id = Column(Integer, primary_key=True)
590 type = Column(Unicode)
591
592 created = Column(DateTime, nullable=False, default=datetime.datetime.now)
593
594 user_id = Column(Integer, ForeignKey('core__users.id'), nullable=False,
595 index=True)
596 seen = Column(Boolean, default=lambda: False, index=True)
597 user = relationship(
598 User,
599 backref=backref('notifications', cascade='all, delete-orphan'))
600
601 __mapper_args__ = {
602 'polymorphic_identity': 'notification',
603 'polymorphic_on': type
604 }
605
606 def __repr__(self):
607 return '<{klass} #{id}: {user}: {subject} ({seen})>'.format(
608 id=self.id,
609 klass=self.__class__.__name__,
610 user=self.user,
611 subject=getattr(self, 'subject', None),
612 seen='unseen' if not self.seen else 'seen')
613
614
615 class CommentNotification(Notification):
616 __tablename__ = 'core__comment_notifications'
617 id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
618
619 subject_id = Column(Integer, ForeignKey(MediaComment.id))
620 subject = relationship(
621 MediaComment,
622 backref=backref('comment_notifications', cascade='all, delete-orphan'))
623
624 __mapper_args__ = {
625 'polymorphic_identity': 'comment_notification'
626 }
627
628
629 class ProcessingNotification(Notification):
630 __tablename__ = 'core__processing_notifications'
631
632 id = Column(Integer, ForeignKey(Notification.id), primary_key=True)
633
634 subject_id = Column(Integer, ForeignKey(MediaEntry.id))
635 subject = relationship(
636 MediaEntry,
637 backref=backref('processing_notifications',
638 cascade='all, delete-orphan'))
639
640 __mapper_args__ = {
641 'polymorphic_identity': 'processing_notification'
642 }
643
644
645 with_polymorphic(
646 Notification,
647 [ProcessingNotification, CommentNotification])
648
649 MODELS = [
650 User, Client, RequestToken, AccessToken, NonceTimestamp, MediaEntry, Tag,
651 MediaTag, MediaComment, Collection, CollectionItem, MediaFile, FileKeynames,
652 MediaAttachmentFile, ProcessingMetaData, Notification, CommentNotification,
653 ProcessingNotification, CommentSubscription]
654
655 """
656 Foundations are the default rows that are created immediately after the tables
657 are initialized. Each entry to this dictionary should be in the format of:
658 ModelConstructorObject:List of Dictionaries
659 (Each Dictionary represents a row on the Table to be created, containing each
660 of the columns' names as a key string, and each of the columns' values as a
661 value)
662
663 ex. [NOTE THIS IS NOT BASED OFF OF OUR USER TABLE]
664 user_foundations = [{'name':u'Joanna', 'age':24},
665 {'name':u'Andrea', 'age':41}]
666
667 FOUNDATIONS = {User:user_foundations}
668 """
669 FOUNDATIONS = {}
670
671 ######################################################
672 # Special, migrations-tracking table
673 #
674 # Not listed in MODELS because this is special and not
675 # really migrated, but used for migrations (for now)
676 ######################################################
677
678 class MigrationData(Base):
679 __tablename__ = "core__migrations"
680
681 name = Column(Unicode, primary_key=True)
682 version = Column(Integer, nullable=False, default=0)
683
684 ######################################################
685
686
687 def show_table_init(engine_uri):
688 if engine_uri is None:
689 engine_uri = 'sqlite:///:memory:'
690 from sqlalchemy import create_engine
691 engine = create_engine(engine_uri, echo=True)
692
693 Base.metadata.create_all(engine)
694
695
696 if __name__ == '__main__':
697 from sys import argv
698 print repr(argv)
699 if len(argv) == 2:
700 uri = argv[1]
701 else:
702 uri = None
703 show_table_init(uri)