Indexing improved and migrator.

Squidex · Mar 4, 2019 · b9f78ec · b9f78ec
1 parent 8834cfb
commit b9f78ec
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 15 deletions.
diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs
@@ -9,6 +9,7 @@
 using System.IO;
 using System.IO.Compression;
 using System.Threading.Tasks;
+using Lucene.Net.Index;
 using Squidex.Infrastructure.Assets;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -18,7 +19,7 @@ public static class PersistenceHelper
         private const string ArchiveFile = "Archive.zip";
         private const string LockFile = "write.lock";
 
-        public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory)
+        public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory, IndexCommit commit)
         {
             using (var fileStream = new FileStream(
                    Path.Combine(directory.FullName, ArchiveFile),
@@ -30,8 +31,10 @@ public static async Task UploadDirectoryAsync(this IAssetStore assetStore, Direc
             {
                 using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
                 {
-                    foreach (var file in directory.GetFiles())
+                    foreach (var fileName in commit.FileNames)
                     {
+                        var file = new FileInfo(Path.Combine(directory.FullName, fileName));
+
                         try
                         {
                             if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) &&

diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs
@@ -37,11 +37,15 @@ public sealed class TextIndexerGrain : GrainOfGuid, ITextIndexerGrain
         private const string MetaDraft = "_dd";
         private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30);
         private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version);
+        private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(MetaDraft, "1"));
+        private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(MetaDraft, "0"));
+        private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
         private readonly IAssetStore assetStore;
         private IDisposable timer;
         private DirectoryInfo directory;
         private IndexWriter indexWriter;
         private IndexReader indexReader;
+        private IndexSearcher indexSearcher;
         private QueryParser queryParser;
         private HashSet<string> currentLanguages;
         private long updates;
@@ -64,8 +68,18 @@ protected override async Task OnActivateAsync(Guid key)
 
             await assetStore.DownloadAsync(directory);
 
-            indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer));
-            indexReader = indexWriter.GetReader(true);
+            var config = new IndexWriterConfig(Version, Analyzer)
+            {
+                IndexDeletionPolicy = snapshotter
+            };
+
+            indexWriter = new IndexWriter(FSDirectory.Open(directory), config);
+
+            if (indexWriter.NumDocs > 0)
+            {
+                indexReader = indexWriter.GetReader(false);
+                indexSearcher = new IndexSearcher(indexReader);
+            }
         }
 
         public Task DeleteAsync(Guid id)
@@ -78,11 +92,9 @@ public Task DeleteAsync(Guid id)
         public Task IndexAsync(Guid id, J<IndexData> data)
         {
             var docId = id.ToString();
-            var docDraft = data.Value.IsDraft.ToString();
+            var docDraft = data.Value.IsDraft ? "1" : "0";
             var docKey = $"{docId}_{docDraft}";
 
-            var query = new BooleanQuery();
-
             indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
 
             var languages = new Dictionary<string, StringBuilder>();
@@ -165,9 +177,9 @@ public Task<List<Guid>> SearchAsync(string queryText, SearchContext context)
 
                 if (indexReader != null)
                 {
-                    var filter = new TermsFilter(new Term(MetaDraft, context.IsDraft.ToString()));
+                    var filter = context.IsDraft ? DraftFilter : NoDraftFilter;
 
-                    var hits = new IndexSearcher(indexReader).Search(query, filter, MaxResults).ScoreDocs;
+                    var hits = indexSearcher.Search(query, filter, MaxResults).ScoreDocs;
 
                     foreach (var hit in hits)
                     {
@@ -236,13 +248,22 @@ public async Task FlushAsync()
         {
             if (updates > 0 && indexWriter != null)
             {
-                indexWriter.Flush(true, true);
                 indexWriter.Commit();
+                indexWriter.Flush(true, true);
 
                 indexReader?.Dispose();
-                indexReader = indexWriter.GetReader(true);
+                indexReader = indexWriter.GetReader(false);
+                indexSearcher = new IndexSearcher(indexReader);
 
-                await assetStore.UploadDirectoryAsync(directory);
+                var commit = snapshotter.Snapshot();
+                try
+                {
+                    await assetStore.UploadDirectoryAsync(directory, commit);
+                }
+                finally
+                {
+                    snapshotter.Release(commit);
+                }
 
                 updates = 0;
             }

diff --git a/tools/Migrate_01/MigrationPath.cs b/tools/Migrate_01/MigrationPath.cs
@@ -16,7 +16,7 @@ namespace Migrate_01
 {
     public sealed class MigrationPath : IMigrationPath
     {
-        private const int CurrentVersion = 14;
+        private const int CurrentVersion = 15;
         private readonly IServiceProvider serviceProvider;
 
         public MigrationPath(IServiceProvider serviceProvider)
@@ -72,7 +72,8 @@ private IEnumerable<IMigration> ResolveMigrators(int version)
             }
 
             // Version 11: Introduce content drafts.
-            if (version < 11)
+            // Version 15: Introduce custom full text search actors.
+            if (version < 15)
             {
                 yield return serviceProvider.GetService<DeleteContentCollections>();
                 yield return serviceProvider.GetRequiredService<RebuildContents>();

diff --git a/tools/Migrate_01/Migrations/DeleteContentCollections.cs b/tools/Migrate_01/Migrations/DeleteContentCollections.cs
@@ -24,6 +24,8 @@ public async Task UpdateAsync()
         {
             await database.DropCollectionAsync("States_Contents");
             await database.DropCollectionAsync("States_Contents_Archive");
+            await database.DropCollectionAsync("State_Content_Draft");
+            await database.DropCollectionAsync("State_Content_Published");
         }
     }
 }
diff --git a/tools/Migrate_01/Rebuilder.cs b/tools/Migrate_01/Rebuilder.cs
@@ -96,7 +96,11 @@ private async Task RebuildManyAsync(string filter, Func<Guid, Task> action)
         {
             var handledIds = new HashSet<Guid>();
 
-            var worker = new ActionBlock<Guid>(action, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 32 });
+            var worker = new ActionBlock<Guid>(action,
+                new ExecutionDataflowBlockOptions
+                {
+                    MaxDegreeOfParallelism = 32
+                });
 
             await eventStore.QueryAsync(async storedEvent =>
             {