Run multiple batches.

Squidex · Jul 24, 2023 · 8693e19 · 8693e19
1 parent a9c77e4
commit 8693e19
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 32 deletions.
diff --git a/cli/Squidex.CLI/Directory.Build.props b/cli/Squidex.CLI/Directory.Build.props
@@ -15,6 +15,6 @@
     <PackageTags>Squidex HeadlessCMS</PackageTags>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>
-    <Version>11.0</Version>
+    <Version>11.1</Version>
   </PropertyGroup>
 </Project>
diff --git a/cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs b/cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs
@@ -131,53 +131,72 @@ public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession s
                     await uploader.CompleteAsync();
                 }
 
-                var request = new BulkUpdateAssetsDto();
+                // Use separate batches to not cause issues with older Squidex version.
+                var annotateBatch = new BulkUpdateAssetsDto
+                {
+                    Jobs = new List<BulkUpdateAssetsJobDto>()
+                };
 
                 foreach (var asset in model.Assets)
                 {
-                    var parentId = await sync.Folders.GetIdAsync(asset.FolderPath);
-
-                    request.Jobs.Add(asset.ToMove(parentId));
-                    request.Jobs.Add(asset.ToAnnotate());
+                    annotateBatch.Jobs.Add(asset.ToAnnotate());
                 }
 
-                var assetIndex = 0;
+                await ExecuteBatchAsync(session, batchIndex, model, annotateBatch, "Annotating");
 
-                var results = await session.Client.Assets.BulkUpdateAssetsAsync(request);
+                var moveBatch = new BulkUpdateAssetsDto
+                {
+                    Jobs = new List<BulkUpdateAssetsJobDto>()
+                };
 
                 foreach (var asset in model.Assets)
                 {
-                    // We create wo commands per asset.
-                    var result1 = results.FirstOrDefault(x => x.JobIndex == (assetIndex * 2));
-                    var result2 = results.FirstOrDefault(x => x.JobIndex == (assetIndex * 2) + 1);
-
-                    log.StepStart($"Upserting #{batchIndex}/{assetIndex}");
-
-                    if (result1?.Error != null)
-                    {
-                        log.StepFailed(result1.Error.ToString());
-                    }
-                    else if (result2?.Error != null)
-                    {
-                        log.StepFailed(result2.Error.ToString());
-                    }
-                    else if (result1?.Id != null && result2?.Id != null)
-                    {
-                        log.StepSuccess();
-                    }
-                    else
-                    {
-                        log.StepSkipped("Unknown Reason");
-                    }
+                    var parentId = await sync.Folders.GetIdAsync(asset.FolderPath);
 
-                    assetIndex++;
+                    moveBatch.Jobs.Add(asset.ToMove(parentId));
                 }
+
+                await ExecuteBatchAsync(session, batchIndex, model, moveBatch, "Moving");
             }
 
             batchIndex++;
         }
     }
 
+    private async Task ExecuteBatchAsync(ISession session, int batchIndex, AssetsModel model, BulkUpdateAssetsDto request, string name)
+    {
+        var index = 0;
+        var results = await session.Client.Assets.BulkUpdateAssetsAsync(request);
+
+        foreach (var asset in model.Assets)
+        {
+            // We create wo commands per asset.
+            var result1 = results.FirstOrDefault(x => x.JobIndex == (index * 2));
+            var result2 = results.FirstOrDefault(x => x.JobIndex == (index * 2) + 1);
+
+            log.StepStart($"{name} #{batchIndex}/{index}");
+
+            if (result1?.Error != null)
+            {
+                log.StepFailed(result1.Error.ToString());
+            }
+            else if (result2?.Error != null)
+            {
+                log.StepFailed(result2.Error.ToString());
+            }
+            else if (result1?.Id != null && result2?.Id != null)
+            {
+                log.StepSuccess();
+            }
+            else
+            {
+                log.StepSkipped("Unknown Reason");
+            }
+
+            index++;
+        }
+    }
+
     private static IEnumerable<IFile> GetFiles(IFileSystem fs)
     {
         foreach (var file in fs.GetFiles(new FilePath("assets"), ".json"))

diff --git a/...quidex.CLI/Squidex.CLI.Core/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs b/...quidex.CLI/Squidex.CLI.Core/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs
@@ -125,6 +125,8 @@ public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession s
 
         var mapper = new Extensions.Mapper(session.Url, session.App, options.Languages);
 
+        var batchIndex = 0;
+
         foreach (var (file, model) in models)
         {
             if (model?.Contents?.Count > 0)
@@ -151,7 +153,7 @@ public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession s
                 {
                     var result = results.Find(x => x.JobIndex == contentIndex);
 
-                    log.StepStart($"Upserting #{contentIndex}");
+                    log.StepStart($"Upserting #{batchIndex}/{contentIndex}");
 
                     if (result?.Error != null)
                     {
@@ -183,6 +185,8 @@ await log.DoSafeAsync($"Saving {file.Name}", async () =>
                     });
                 }
             }
+
+            batchIndex++;
         }
     }