Permanent deletion (#1144)

* Delete permanent. * Permanent deletion.
Squidex · Nov 24, 2024 · a7e44b2 · a7e44b2
1 parent 2b8c93d
commit a7e44b2
Show file tree

Hide file tree

Showing 118 changed files with 1,217 additions and 349 deletions.
diff --git a/.../src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/ContentFieldProperty.cs b/.../src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/ContentFieldProperty.cs
@@ -40,7 +40,9 @@ protected override JsValue? CustomValue
             {
                 this.value = value;
 
+#pragma warning disable MA0143 // Primary constructor parameters should be readonly
                 contentValue = newContentValue;
+#pragma warning restore MA0143 // Primary constructor parameters should be readonly
                 contentField.MarkChanged();
 
                 isChanged = true;

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs
@@ -127,6 +127,12 @@ public IAsyncEnumerable<Content> QueryScheduledWithoutDataAsync(Instant now,
         return queryScheduled.QueryAsync(now, ct);
     }
 
+    public IAsyncEnumerable<DomainId> StreamIds(DomainId appId, DomainId schemaId,
+        CancellationToken ct)
+    {
+        return queryAsStream.StreamAllIds(appId, schemaId, ct);
+    }
+
     public async Task DeleteAppAsync(DomainId appId,
         CancellationToken ct)
     {
@@ -136,6 +142,15 @@ public async Task DeleteAppAsync(DomainId appId,
         }
     }
 
+    public async Task DeleteSchemaAsync(DomainId schemaId,
+        CancellationToken ct)
+    {
+        using (Telemetry.Activities.StartActivity("MongoContentCollection/DeleteSchemaAsync"))
+        {
+            await Collection.DeleteManyAsync(Filter.Eq(x => x.IndexedSchemaId, schemaId), ct);
+        }
+    }
+
     public async Task<IResultList<Content>> QueryAsync(App app, List<Schema> schemas, Q q,
         CancellationToken ct)
     {

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs
@@ -81,6 +81,12 @@ public IAsyncEnumerable<Content> StreamScheduledWithoutDataAsync(Instant now, Se
         return GetCollection(scope).QueryScheduledWithoutDataAsync(now, ct);
     }
 
+    public IAsyncEnumerable<DomainId> StreamIds(DomainId appId, DomainId schemaId, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return GetCollection(scope).StreamIds(appId, schemaId, ct);
+    }
+
     public Task<IResultList<Content>> QueryAsync(App app, List<Schema> schemas, Q q, SearchScope scope,
         CancellationToken ct = default)
     {

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs
@@ -103,4 +103,16 @@ public async IAsyncEnumerable<Content> StreamScheduledWithoutDataAsync(Instant n
             }
         }
     }
+
+    public async IAsyncEnumerable<DomainId> StreamIds(DomainId appId, DomainId schemaId, SearchScope scope,
+       [EnumeratorCancellation] CancellationToken ct = default)
+    {
+        foreach (var shard in Shards)
+        {
+            await foreach (var id in shard.StreamIds(appId, schemaId, scope, ct))
+            {
+                yield return id;
+            }
+        }
+    }
 }
diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryAsStream.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryAsStream.cs
@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using System.Runtime.CompilerServices;
+using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Infrastructure;
@@ -14,6 +15,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations;
 
 public sealed class QueryAsStream : OperationBase
 {
+    public sealed class IdOnly
+    {
+        [BsonElement("id")]
+        public DomainId Id { get; set; }
+    }
+
     public async IAsyncEnumerable<Content> StreamAll(DomainId appId, HashSet<DomainId>? schemaIds,
         [EnumeratorCancellation] CancellationToken ct)
     {
@@ -31,6 +38,26 @@ public async IAsyncEnumerable<Content> StreamAll(DomainId appId, HashSet<DomainI
         }
     }
 
+    public async IAsyncEnumerable<DomainId> StreamAllIds(DomainId appId, DomainId schemaId,
+        [EnumeratorCancellation] CancellationToken ct)
+    {
+        var filter = CreateFilter(appId, [schemaId]);
+
+        // Only query the ID from the database to improve performance.
+        var projection = Builders<MongoContentEntity>.Projection.Include(x => x.Id);
+
+        using (var cursor = await Collection.Find(filter).Project<IdOnly>(projection).ToCursorAsync(ct))
+        {
+            while (await cursor.MoveNextAsync(ct))
+            {
+                foreach (var entity in cursor.Current)
+                {
+                    yield return entity.Id;
+                }
+            }
+        }
+    }
+
     private static FilterDefinition<MongoContentEntity> CreateFilter(DomainId appId, HashSet<DomainId>? schemaIds)
     {
         var filters = new List<FilterDefinition<MongoContentEntity>>

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemaRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemaRepository.cs
@@ -39,6 +39,12 @@ Task IDeleter.DeleteAppAsync(App app,
         return Collection.DeleteManyAsync(Filter.Eq(x => x.IndexedAppId, app.Id), ct);
     }
 
+    Task IDeleter.DeleteSchemaAsync(App app, Schema schema,
+        CancellationToken ct)
+    {
+        return Collection.DeleteManyAsync(Filter.Eq(x => x.IndexedId, schema.Id), ct);
+    }
+
     public async Task<List<Schema>> QueryAllAsync(DomainId appId, CancellationToken ct = default)
     {
         using (Telemetry.Activities.StartActivity("MongoSchemaRepository/QueryAllAsync"))

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemasHash.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemasHash.cs
@@ -31,10 +31,10 @@ protected override string CollectionName()
         return "SchemasHash";
     }
 
-    async Task IDeleter.DeleteAppAsync(App app,
+    Task IDeleter.DeleteAppAsync(App app,
         CancellationToken ct)
     {
-        await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
+        return Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
     }
 
     public Task On(IEnumerable<Envelope<IEvent>> events)

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs
@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Core.Schemas;
 using Squidex.Domain.Apps.Entities.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Infrastructure;
@@ -46,12 +47,21 @@ public async Task ExecuteAsync(IndexCommand[] commands,
         return Shard(app.Id).SearchAsync(app, query, scope, ct);
     }
 
-    public async Task DeleteAppAsync(App app,
+    async Task IDeleter.DeleteAppAsync(App app,
         CancellationToken ct)
     {
         if (Shard(app.Id) is IDeleter shard)
         {
             await shard.DeleteAppAsync(app, ct);
         }
     }
+
+    async Task IDeleter.DeleteSchemaAsync(App app, Schema schema,
+        CancellationToken ct)
+    {
+        if (Shard(app.Id) is IDeleter shard)
+        {
+            await shard.DeleteSchemaAsync(app, schema, ct);
+        }
+    }
 }
diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs
@@ -9,6 +9,7 @@
 using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Core.Schemas;
 using Squidex.Domain.Apps.Entities.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Infrastructure;
@@ -57,10 +58,16 @@ protected override string CollectionName()
         return $"TextIndex2{shardKey}";
     }
 
-    async Task IDeleter.DeleteAppAsync(App app,
+    Task IDeleter.DeleteAppAsync(App app,
         CancellationToken ct)
     {
-        await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
+        return Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
+    }
+
+    Task IDeleter.DeleteSchemaAsync(App app, Schema schema,
+        CancellationToken ct)
+    {
+        return Collection.DeleteManyAsync(Filter.Eq(x => x.SchemaId, schema.Id), ct);
     }
 
     public async virtual Task ExecuteAsync(IndexCommand[] commands,

diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs
@@ -8,14 +8,20 @@
 using MongoDB.Bson.Serialization;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Core.Schemas;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Repositories;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
 
 namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
 
-public sealed class MongoTextIndexerState(IMongoDatabase database) : MongoRepositoryBase<TextContentState>(database), ITextIndexerState, IDeleter
+public sealed class MongoTextIndexerState(
+    IMongoDatabase database,
+    IContentRepository contentRepository)
+    : MongoRepositoryBase<TextContentState>(database), ITextIndexerState, IDeleter
 {
     static MongoTextIndexerState()
     {
@@ -30,6 +36,8 @@ static MongoTextIndexerState()
         });
     }
 
+    int IDeleter.Order => -2000;
+
     protected override string CollectionName()
     {
         return "TextIndexerState";
@@ -46,6 +54,20 @@ async Task IDeleter.DeleteAppAsync(App app,
         await Collection.DeleteManyAsync(filter, ct);
     }
 
+    async Task IDeleter.DeleteSchemaAsync(App app, Schema schema,
+        CancellationToken ct)
+    {
+        var ids = contentRepository.StreamIds(app.Id, schema.Id, SearchScope.All, ct).Batch(1000, ct);
+
+        await foreach (var batch in ids.WithCancellation(ct))
+        {
+            var filter =
+                Filter.In(x => x.UniqueContentId, batch.Select(x => new UniqueContentId(app.Id, x)));
+
+            await Collection.DeleteManyAsync(filter, ct);
+        }
+    }
+
     public async Task<Dictionary<UniqueContentId, TextContentState>> GetAsync(HashSet<UniqueContentId> ids,
         CancellationToken ct = default)
     {

diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/AppPermanentDeleter.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/AppPermanentDeleter.cs
@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using Microsoft.Extensions.Options;
 using Squidex.Domain.Apps.Entities.Apps.DomainObject;
 using Squidex.Domain.Apps.Events.Apps;
 using Squidex.Infrastructure;
@@ -14,9 +15,15 @@
 
 namespace Squidex.Domain.Apps.Entities.Apps;
 
-public sealed class AppPermanentDeleter(IEnumerable<IDeleter> deleters, IDomainObjectFactory factory, TypeRegistry typeRegistry) : IEventConsumer
+public sealed class AppPermanentDeleter(
+    IEnumerable<IDeleter> deleters,
+    IOptions<AppsOptions> options,
+    IDomainObjectFactory factory,
+    TypeRegistry typeRegistry)
+    : IEventConsumer
 {
     private readonly IEnumerable<IDeleter> deleters = deleters.OrderBy(x => x.Order).ToList();
+    private readonly AppsOptions options = options.Value;
     private readonly HashSet<string> consumingTypes =
         [
             typeRegistry.GetName<IEvent, AppDeleted>(),
@@ -39,8 +46,8 @@ public async Task On(Envelope<IEvent> @event)
 
         switch (@event.Payload)
         {
-            case AppDeleted appArchived:
-                await OnArchiveAsync(appArchived);
+            case AppDeleted appDeleted:
+                await OnDeleteAsync(appDeleted);
                 break;
             case AppContributorRemoved appContributorRemoved:
                 await OnAppContributorRemoved(appContributorRemoved);
@@ -63,17 +70,18 @@ private async Task OnAppContributorRemoved(AppContributorRemoved appContributorR
         }
     }
 
-    private async Task OnArchiveAsync(AppDeleted appArchived)
+    private async Task OnDeleteAsync(AppDeleted appDeleted)
     {
-        using var activity = Telemetry.Activities.StartActivity("RemoveAppFromSystem");
-
-        // Bypass our normal app resolve process, so that we can also retrieve the deleted app.
-        var app = factory.Create<AppDomainObject>(appArchived.AppId.Id);
+        // The user can either remove the app itself or via a global setting for all apps.
+        if (!appDeleted.Permanent && !options.DeletePermanent)
+        {
+            return;
+        }
 
-        await app.EnsureLoadedAsync();
+        using var activity = Telemetry.Activities.StartActivity("RemoveAppFromSystem");
 
-        // If the app does not exist, the version is lower than zero.
-        if (app.Version < 0)
+        var app = await GetAppAsync(appDeleted.AppId.Id);
+        if (app == null)
         {
             return;
         }
@@ -86,4 +94,15 @@ private async Task OnArchiveAsync(AppDeleted appArchived)
             }
         }
     }
+
+    private async Task<AppDomainObject?> GetAppAsync(DomainId appId)
+    {
+        // Bypass our normal resolve process, so that we can also retrieve the deleted app.
+        var app = factory.Create<AppDomainObject>(appId);
+
+        await app.EnsureLoadedAsync();
+
+        // If the app does not exist, the version is lower than zero.
+        return app.Version < 0 ? null : app;
+    }
 }
diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/AppsOptions.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/AppsOptions.cs
@@ -0,0 +1,13 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Domain.Apps.Entities.Apps;
+
+public sealed class AppsOptions
+{
+    public bool DeletePermanent { get; set; }
+}
diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/DeleteApp.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/DeleteApp.cs
@@ -9,4 +9,5 @@ namespace Squidex.Domain.Apps.Entities.Apps.Commands;
 
 public sealed class DeleteApp : AppCommand
 {
+    public bool Permanent { get; set; }
 }
diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs
@@ -122,7 +122,7 @@ public async Task ReadLogAsync(DomainId appId, DateTime fromDate, DateTime toDat
         }
         finally
         {
-            await writer.FlushAsync();
+            await writer.FlushAsync(ct);
         }
     }
 

diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentEventDeleter.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentEventDeleter.cs
@@ -0,0 +1,35 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Core.Schemas;
+using Squidex.Domain.Apps.Entities.Contents.Repositories;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.EventSourcing;
+
+namespace Squidex.Domain.Apps.Entities.Contents;
+
+public sealed class ContentEventDeleter(IContentRepository contentRepository, IEventStore eventStore) : IDeleter
+{
+    public int Order => -1000;
+
+    public Task DeleteAppAsync(App app, CancellationToken ct)
+    {
+        return Task.CompletedTask;
+    }
+
+    public async Task DeleteSchemAsync(App app, Schema schema,
+        CancellationToken ct)
+    {
+        await foreach (var id in contentRepository.StreamIds(app.Id, schema.Id, SearchScope.All, ct))
+        {
+            var streamFilter = StreamFilter.Prefix($"content-{DomainId.Combine(app.Id, id)}");
+
+            await eventStore.DeleteAsync(streamFilter, ct);
+        }
+    }
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,4 +9,5 @@ namespace Squidex.Domain.Apps.Entities.Apps.Commands; @@
     public sealed class DeleteApp : AppCommand
     {
+        public bool Permanent { get; set; }
     }