From bc75a6532712e4b9b0f6d8b5bbd93db3ef58335d Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 1 Oct 2015 15:59:20 +0200 Subject: [PATCH] Add Graveyard model This adds the Graveyard model which is used when a model is deleted, it stores the important "shell" information on the model so it can hard-delete the real object. It also remaps the GenericModelReference references to the new Graveyard model. This also moves the soft deletion setting from __model_args__ to "deletion_mode" on the model. --- mediagoblin/db/base.py | 87 +++++++++++++++++----------------- mediagoblin/db/migrations.py | 73 +++++++---------------------- mediagoblin/db/models.py | 90 ++++++++++++++++++++++++++---------- 3 files changed, 126 insertions(+), 124 deletions(-) diff --git a/mediagoblin/db/base.py b/mediagoblin/db/base.py index a425343b..a62cbebc 100644 --- a/mediagoblin/db/base.py +++ b/mediagoblin/db/base.py @@ -13,8 +13,6 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import datetime - from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import inspect @@ -30,11 +28,7 @@ class GMGTableBase(object): HARD_DELETE = "hard-deletion" SOFT_DELETE = "soft-deletion" - __default_model_args__ = { - "deletion": HARD_DELETE, - "soft_deletion_field": "deleted", - "soft_deletion_retain": ("id",) - } + deletion_mode = HARD_DELETE @property def _session(self): @@ -47,11 +41,6 @@ class GMGTableBase(object): if not DISABLE_GLOBALS: query = Session.query_property() - def get_model_arg(self, argument): - model_args = self.__default_model_args__.copy() - model_args.update(getattr(self, "__model_args__", {})) - return model_args.get(argument) - def get(self, key): return getattr(self, key) @@ -70,42 +59,54 @@ class GMGTableBase(object): else: sess.flush() - def delete(self, commit=True): + def delete(self, commit=True, deletion=None): """ Delete the object either using soft or hard deletion """ - if self.get_model_arg("deletion") == self.HARD_DELETE: - return self.hard_delete(commit) - elif self.get_model_arg("deletion") == self.SOFT_DELETE: - return self.soft_delete(commit) + # Get the setting in the model args if none has been specified. + if deletion is None: + deletion = self.deletion_mode + + # Hand off to the correct deletion function. + if deletion == self.HARD_DELETE: + return self.hard_delete(commit=commit) + elif deletion == self.SOFT_DELETE: + return self.soft_delete(commit=commit) else: raise ValueError( - "__model_args__['deletion'] is an invalid value %s" % ( - self.get_model_arg("deletion") - )) + "Invalid deletion mode {mode!r}".format( + mode=deletion + ) + ) def soft_delete(self, commit): - # Find the deletion field - field_name = self.get_model_arg("soft_deletion_field") - - # We can't use self.__table__.columns as it only shows it of the - # current model and no parent if polymorphism is being used. This - # will cause problems for example for the User model. - if field_name not in dir(type(self)): - raise ValueError("Cannot find soft_deletion_field") - - # Store a value in the deletion field - setattr(self, field_name, datetime.datetime.utcnow()) - - # Iterate through the fields and remove data - retain_fields = self.get_model_arg("soft_deletion_retain") - for field_name in self.__table__.columns.keys(): - # should we skip this field? - if field_name in retain_fields: - continue - - setattr(self, field_name, None) - - # Save the changes - self.save(commit) + # Create the graveyard version of this model + # Importing this here due to cyclic imports + from mediagoblin.db.models import User, Graveyard, GenericModelReference + tombstone = Graveyard() + if getattr(self, "public_id", None) is not None: + tombstone.public_id = self.public_id + + # This is a special case, we don't want to save any actor if the thing + # being soft deleted is a User model as this would create circular + # ForeignKeys + if not isinstance(self, User): + tombstone.actor = User.query.filter_by( + id=self.actor + ).first() + tombstone.object_type = self.object_type + tombstone.save() + + # There will be a lot of places where the GenericForeignKey will point + # to the model, we want to remap those to our tombstone. + gmrs = GenericModelReference.query.filter_by( + obj_pk=self.id, + model_type=self.__tablename__ + ).update({ + "obj_pk": tombstone.id, + "model_type": tombstone.__tablename__, + }) + + # Now we can go ahead and actually delete the model. + return self.hard_delete(commit=commit) def hard_delete(self, commit): """Delete the object and commit the change immediately by default""" diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 019eb338..36ad736a 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -1821,69 +1821,28 @@ def federation_actor(db): # commit changes to db. db.commit() -@RegisterMigration(39, MIGRATIONS) -def federation_soft_deletion(db): - """ Introduces soft deletion to models +class Graveyard_V0(declarative_base()): + """ Where models come to die """ + __tablename__ = "core__graveyard" - This adds a deleted DateTime column which represents if the model is - deleted and if so, when. With this change comes changes on the models - that soft delete the models rather than the previous hard deletion. - """ - metadata = MetaData(bind=db.bind) + id = Column(Integer, primary_key=True) + public_id = Column(Unicode, nullable=True, unique=True) - # User Model - user_table = inspect_table(metadata, "core__users") - user_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - user_deleted_column.create(user_table) + deleted = Column(DateTime, nullable=False) + object_type = Column(Unicode, nullable=False) - # MediaEntry - media_entry_table = inspect_table(metadata, "core__media_entries") - me_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - me_deleted_column.create(media_entry_table) + actor_id = Column(Integer, ForeignKey(GenericModelReference_V0.id)) - # MediaComment - media_comment_table = inspect_table(metadata, "core__media_comments") - mc_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - mc_deleted_column.create(media_comment_table) - - # Collection - collection_table = inspect_table(metadata, "core__collections") - collection_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - collection_deleted_column.create(collection_table) +@RegisterMigration(39, MIGRATIONS) +def federation_graveyard(db): + """ Introduces soft deletion to models - # Generator - generator_table = inspect_table(metadata, "core__generators") - generator_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - generator_deleted_column.create(generator_table) + This adds a Graveyard model which is used to copy (soft-)deleted models to. + """ + metadata = MetaData(bind=db.bind) - # Activity - activity_table = inspect_table(metadata, "core__activities") - activity_deleted_column = Column( - "deleted", - DateTime, - nullable=True - ) - activity_deleted_column.create(activity_table) + # Create the graveyard table + Graveyard_V0.__table__.create(db.bind) # Commit changes to the db db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index d7ddc55d..73f3c8ce 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -243,7 +243,6 @@ class User(Base, UserMixin): created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) location = Column(Integer, ForeignKey("core__locations.id")) @@ -255,11 +254,25 @@ class User(Base, UserMixin): 'polymorphic_on': type, } - __model_args__ = { - 'deletion': Base.SOFT_DELETE, - } + deletion_mode = Base.SOFT_DELETE + + def soft_delete(self, *args, **kwargs): + # Find all the Collections and delete those + for collection in Collection.query.filter_by(actor=self.id): + collection.delete(**kwargs) + + # Find all the comments and delete those too + for comment in MediaComment.query.filter_by(actor=self.id): + comment.delete(**kwargs) + + # Find all the activities and delete those too + for activity in Activity.query.filter_by(actor=self.id): + activity.delete(**kwargs) - def delete(self, **kwargs): + super(User, self).soft_delete(*args, **kwargs) + + + def delete(self, *args, **kwargs): """Deletes a User and all related entries/comments/files/...""" # Collections get deleted by relationships. @@ -276,7 +289,7 @@ class User(Base, UserMixin): # Delete user, pass through commit=False/True in kwargs username = self.username - super(User, self).delete(**kwargs) + super(User, self).delete(*args, **kwargs) _log.info('Deleted user "{0}" account'.format(username)) def has_privilege(self, privilege, allow_admin=True): @@ -521,7 +534,6 @@ class MediaEntry(Base, MediaEntryMixin): created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow, index=True) updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) fail_error = Column(Unicode) fail_metadata = Column(JSONEncoded) @@ -536,6 +548,8 @@ class MediaEntry(Base, MediaEntryMixin): UniqueConstraint('actor', 'slug'), {}) + deletion_mode = Base.SOFT_DELETE + get_actor = relationship(User) media_files_helper = relationship("MediaFile", @@ -673,6 +687,13 @@ class MediaEntry(Base, MediaEntryMixin): id=self.id, title=safe_title) + def soft_delete(self, *args, **kwargs): + # Find all of the media comments for this and delete them + for comment in MediaComment.query.filter_by(media_entry=self.id): + comment.delete(*args, **kwargs) + + super(MediaEntry, self).soft_delete(*args, **kwargs) + def delete(self, del_orphan_tags=True, **kwargs): """Delete MediaEntry and all related files/attachments/comments @@ -915,7 +936,6 @@ class MediaComment(Base, MediaCommentMixin): Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) actor = Column(Integer, ForeignKey(User.id), nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) content = Column(UnicodeText, nullable=False) location = Column(Integer, ForeignKey("core__locations.id")) get_location = relationship("Location", lazy="joined") @@ -941,9 +961,7 @@ class MediaComment(Base, MediaCommentMixin): lazy="dynamic", cascade="all, delete-orphan")) - __model_args__ = { - "deletion": Base.SOFT_DELETE, - } + deletion_mode = Base.SOFT_DELETE def serialize(self, request): """ Unserialize to python dictionary for API """ @@ -1021,7 +1039,6 @@ class Collection(Base, CollectionMixin): created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow, index=True) updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) description = Column(UnicodeText) actor = Column(Integer, ForeignKey(User.id), nullable=False) num_items = Column(Integer, default=0) @@ -1043,9 +1060,7 @@ class Collection(Base, CollectionMixin): UniqueConstraint("actor", "slug"), {}) - __model_args__ = { - "delete": Base.SOFT_DELETE, - } + deletion_mode = Base.SOFT_DELETE # These are the types, It's strongly suggested if new ones are invented they # are prefixed to ensure they're unique from other types. Any types used in @@ -1438,12 +1453,9 @@ class Generator(Base): name = Column(Unicode, nullable=False) published = Column(DateTime, default=datetime.datetime.utcnow) updated = Column(DateTime, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) object_type = Column(Unicode, nullable=False) - __model_args__ = { - "deletion": Base.SOFT_DELETE, - } + deletion_mode = Base.SOFT_DELETE def __repr__(self): return "<{klass} {name}>".format( @@ -1485,7 +1497,6 @@ class Activity(Base, ActivityMixin): nullable=False) published = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) - deleted = Column(DateTime, nullable=True) verb = Column(Unicode, nullable=False) content = Column(Unicode, nullable=True) @@ -1511,9 +1522,7 @@ class Activity(Base, ActivityMixin): cascade="all, delete-orphan")) get_generator = relationship(Generator) - __model_args__ = { - "deletion": Base.SOFT_DELETE, - } + deletion_mode = Base.SOFT_DELETE def __repr__(self): if self.content is None: @@ -1532,6 +1541,39 @@ class Activity(Base, ActivityMixin): self.updated = datetime.datetime.now() super(Activity, self).save(*args, **kwargs) +class Graveyard(Base): + """ Where models come to die """ + __tablename__ = "core__graveyard" + + id = Column(Integer, primary_key=True) + public_id = Column(Unicode, nullable=True, unique=True) + + deleted = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) + object_type = Column(Unicode, nullable=False) + + # This could either be a deleted actor or a real actor, this must be + # nullable as it we shouldn't have it set for deleted actor + actor_id = Column(Integer, ForeignKey(GenericModelReference.id)) + actor_helper = relationship(GenericModelReference) + actor = association_proxy("actor_helper", "get_object", + creator=GenericModelReference.find_or_new) + + def __repr__(self): + return "<{klass} deleted {obj_type}>".format( + klass=type(self).__name__, + obj_type=self.object_type + ) + + def serialize(self, request): + return { + "id": self.public_id, + "objectType": self.object_type, + "actor": self.actor(), + "published": self.deleted, + "updated": self.deleted, + "deleted": self.deleted + } + with_polymorphic( Notification, [ProcessingNotification, CommentNotification]) @@ -1543,7 +1585,7 @@ MODELS = [ ProcessingNotification, Client, CommentSubscription, ReportBase, CommentReport, MediaReport, UserBan, Privilege, PrivilegeUserAssociation, RequestToken, AccessToken, NonceTimestamp, Activity, Generator, Location, - GenericModelReference] + GenericModelReference, Graveyard] """ Foundations are the default rows that are created immediately after the tables -- 2.25.1