diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/App_Assets.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/App_Assets.cs index 14ce1c28..da482421 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/App_Assets.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/App_Assets.cs @@ -6,6 +6,8 @@ // ========================================================================== using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -74,7 +76,7 @@ public async Task Import(ImportArguments arguments) { var existing = existings.Items.First(); - log.WriteLine($"Updating: {file.FullName}"); + log.WriteLine($"Uploading: {file.FullName}"); await assets.PutAssetContentAsync(session.App, existing.Id, fileParameter); @@ -82,7 +84,7 @@ public async Task Import(ImportArguments arguments) } else { - log.WriteLine($"Uploading: {file.FullName}"); + log.WriteLine($"Uploading New: {file.FullName}"); var result = await assets.PostAssetAsync(session.App, parentId, duplicate: arguments.Duplicate, file: fileParameter); @@ -110,6 +112,56 @@ public async Task Import(ImportArguments arguments) } } + [Command(Name = "export", Description = "Export all files to the source folder.")] + public async Task Export(ImportArguments arguments) + { + var session = configuration.StartSession(); + + var assets = session.Assets; + + using (var fs = FileSystems.Create(arguments.Path)) + { + var folderTree = new FolderTree(session); + var folderNames = new HashSet(); + + var parentId = await folderTree.GetIdAsync(arguments.TargetFolder); + + var downloadPipeline = new DownloadPipeline(session, log, fs) + { + FilePathProviderAsync = async asset => + { + var assetFolder = await folderTree.GetPathAsync(asset.ParentId); + var assetPath = asset.FileName; + + if (!string.IsNullOrWhiteSpace(assetFolder)) + { + assetPath = Path.Combine(assetFolder, assetPath); + } + + if (!folderNames.Add(assetPath)) + { + assetPath = Path.Combine(assetFolder, $"{asset.Id}_{asset.FileName}"); + } + + return FilePath.Create(assetPath); + } + }; + + await assets.GetAllByQueryAsync(session.App, async asset => + { + await downloadPipeline.DownloadAsync(asset); + }, + new AssetQuery + { + ParentId = parentId + }); + + await downloadPipeline.CompleteAsync(); + + log.WriteLine("> Export completed"); + } + } + [Validator(typeof(Validator))] public sealed class ImportArguments : IArgumentModel { @@ -130,6 +182,24 @@ public Validator() } } } + + [Validator(typeof(Validator))] + public sealed class ExportArguments : IArgumentModel + { + [Operand(Name = "folder", Description = "The source folder.")] + public string Path { get; set; } + + [Option(ShortName = "t", LongName = "target", Description = "Path to the target folder.")] + public string SourceFolder { get; set; } + + public sealed class Validator : AbstractValidator + { + public Validator() + { + RuleFor(x => x.Path).NotEmpty(); + } + } + } } } } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/FileSystem/FilePath.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/FileSystem/FilePath.cs index dca4101a..99837017 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/FileSystem/FilePath.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/FileSystem/FilePath.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.Collections.Generic; using System.IO; using System.Linq; @@ -15,13 +14,18 @@ public sealed class FilePath { public static readonly FilePath Root = new FilePath(string.Empty); - public IEnumerable Elements { get; } + public string[] Elements { get; } public FilePath(params string[] elements) { Elements = elements; } + public static FilePath Create(string path) + { + return new FilePath(path.Split('/', '\\')); + } + public FilePath Combine(FilePath path) { return new FilePath(Elements.Concat(path.Elements).ToArray()); @@ -29,7 +33,7 @@ public FilePath Combine(FilePath path) public override string ToString() { - return Path.Combine(Elements.ToArray()); + return Path.Combine(Elements); } } } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs index 49b5ca0d..d808589e 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs @@ -40,7 +40,10 @@ public Task CleanupAsync(IFileSystem fs) public async Task ExportAsync(ISyncService sync, SyncOptions options, ISession session) { - var downloadPipeline = new DownloadPipeline(session, log, sync.FileSystem); + var downloadPipeline = new DownloadPipeline(session, log, sync.FileSystem) + { + FilePathProvider = asset => asset.Id.GetBlobPath() + }; var assets = new List(); var assetBatch = 0; @@ -95,7 +98,10 @@ public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession s { if (model?.Assets?.Count > 0) { - var uploader = new UploadPipeline(session, log, sync.FileSystem); + var uploader = new UploadPipeline(session, log, sync.FileSystem) + { + FilePathProvider = asset => asset.Id.GetBlobPath() + }; await uploader.UploadAsync(model.Assets); await uploader.CompleteAsync(); diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/DownloadPipeline.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/DownloadPipeline.cs index 879f583c..a347cede 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/DownloadPipeline.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/DownloadPipeline.cs @@ -17,17 +17,50 @@ namespace Squidex.CLI.Commands.Implementation.Sync.Assets { public sealed class DownloadPipeline { - private readonly ActionBlock pipeline; + private readonly ITargetBlock pipelineStart; + private readonly IDataflowBlock pipelineEnd; + + public Func FilePathProvider { get; set; } + + public Func> FilePathProviderAsync { get; set; } public DownloadPipeline(ISession session, ILogger log, IFileSystem fs) { - pipeline = new ActionBlock(async asset => + var fileNameStep = new TransformBlock(async asset => + { + FilePath path; + + if (FilePathProvider != null) + { + path = FilePathProvider(asset); + } + else if (FilePathProviderAsync != null) + { + path = await FilePathProviderAsync(asset); + } + else + { + path = new FilePath(asset.Id); + } + + return (asset, path); + }, + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1, + BoundedCapacity = 1 + }); + + var downloadStep = new ActionBlock<(AssetDto, FilePath)>(async item => { - var process = $"Downloading {asset.Id}"; + var (asset, path) = item; + + var process = $"Downloading {path}"; try { - var assetFile = fs.GetBlobFile(asset.Id); + var assetFile = fs.GetFile(path); var assetHash = GetFileHash(assetFile, asset); if (assetHash == null || !string.Equals(asset.FileHash, assetHash)) @@ -36,9 +69,9 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs) await using (response.Stream) { - await using (var fileStream = assetFile.OpenWrite()) + await using (var stream = assetFile.OpenWrite()) { - await response.Stream.CopyToAsync(fileStream); + await response.Stream.CopyToAsync(stream); } } @@ -59,6 +92,14 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs) MaxMessagesPerTask = 1, BoundedCapacity = 16 }); + + fileNameStep.LinkTo(downloadStep, new DataflowLinkOptions + { + PropagateCompletion = true + }); + + pipelineStart = fileNameStep; + pipelineEnd = downloadStep; } private static string GetFileHash(IFile file, AssetDto asset) @@ -101,14 +142,14 @@ private static string GetFileHash(IFile file, AssetDto asset) public Task DownloadAsync(AssetDto asset) { - return pipeline.SendAsync(asset); + return pipelineStart.SendAsync(asset); } public Task CompleteAsync() { - pipeline.Complete(); + pipelineEnd.Complete(); - return pipeline.Completion; + return pipelineEnd.Completion; } } } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/Extensions.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/Extensions.cs index 91ee5ebc..199e9877 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/Extensions.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/Extensions.cs @@ -15,7 +15,12 @@ public static class Extensions { public static IFile GetBlobFile(this IFileSystem fs, string id) { - return fs.GetFile(new FilePath("assets", "files", $"{id}.blob")); + return fs.GetFile(GetBlobPath(id)); + } + + public static FilePath GetBlobPath(this string id) + { + return new FilePath("assets", "files", $"{id}.blob"); } public static BulkUpdateAssetsJobDto ToMoveJob(this AssetModel model, string parentId) diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs index 018213c6..52c797ec 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs @@ -16,19 +16,52 @@ namespace Squidex.CLI.Commands.Implementation.Sync.Assets { public sealed class UploadPipeline { - private readonly ActionBlock pipeline; + private readonly ITargetBlock pipelineStart; + private readonly IDataflowBlock pipelineEnd; + + public Func FilePathProvider { get; set; } + + public Func> FilePathProviderAsync { get; set; } public UploadPipeline(ISession session, ILogger log, IFileSystem fs) { var tree = new FolderTree(session); - pipeline = new ActionBlock(async asset => + var fileNameStep = new TransformBlock(async asset => + { + FilePath path; + + if (FilePathProvider != null) + { + path = FilePathProvider(asset); + } + else if (FilePathProviderAsync != null) + { + path = await FilePathProviderAsync(asset); + } + else + { + path = new FilePath(asset.Id); + } + + return (asset, path); + }, + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1, + BoundedCapacity = 1 + }); + + var uploadStep = new ActionBlock<(AssetModel, FilePath)>(async item => { - var process = $"Uploading {asset.Id}"; + var (asset, path) = item; + + var process = $"Uploading {path}"; try { - var assetFile = fs.GetBlobFile(asset.Id); + var assetFile = fs.GetFile(path); await using (var stream = assetFile.OpenRead()) { @@ -56,21 +89,29 @@ public UploadPipeline(ISession session, ILogger log, IFileSystem fs) MaxMessagesPerTask = 1, BoundedCapacity = 16 }); + + fileNameStep.LinkTo(uploadStep, new DataflowLinkOptions + { + PropagateCompletion = true + }); + + pipelineStart = fileNameStep; + pipelineEnd = uploadStep; } public async Task UploadAsync(IEnumerable assets) { foreach (var asset in assets) { - await pipeline.SendAsync(asset); + await pipelineStart.SendAsync(asset); } } public Task CompleteAsync() { - pipeline.Complete(); + pipelineEnd.Complete(); - return pipeline.Completion; + return pipelineEnd.Completion; } } } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs index fd77f63e..1b9b446f 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs @@ -20,7 +20,7 @@ public sealed class ContentsSynchronizer : ISynchronizer private const string Ref = "../__json/contents"; private readonly ILogger log; - public string Name => "contents"; + public string Name => "Contents"; public ContentsSynchronizer(ILogger log) { diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/SyncService.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/SyncService.cs index 66b3fab1..1cc2f312 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/SyncService.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/SyncService.cs @@ -109,29 +109,25 @@ public T Read(IFile file, ILogger log) return JsonConvert.DeserializeObject(jsonText, jsonSerializerSettings); } - public Task WriteWithSchemaAs(IFile file, object sample, string schema) where T : class + public async Task WriteWithSchemaAs(IFile file, object sample, string schema) where T : class { - using (var stream = file.OpenWrite()) + await using (var stream = file.OpenWrite()) { Write(Convert(sample), stream, $"./{schema}.json"); } - - return Task.CompletedTask; } - public Task WriteWithSchema(IFile file, T sample, string schema) where T : class + public async Task WriteWithSchema(IFile file, T sample, string schema) where T : class { - using (var stream = file.OpenWrite()) + await using (var stream = file.OpenWrite()) { Write(sample, stream, $"./{schema}.json"); } - - return Task.CompletedTask; } - public Task WriteJsonSchemaAsync(IFile file) + public async Task WriteJsonSchemaAsync(IFile file) { - using (var stream = file.OpenWrite()) + await using (var stream = file.OpenWrite()) { using (var textWriter = new StreamWriter(stream)) { @@ -147,8 +143,6 @@ public Task WriteJsonSchemaAsync(IFile file) textWriter.Write(json); } } - - return Task.CompletedTask; } public void Write(T value, Stream stream, string schemaRef = null) where T : class diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Workflows/WorkflowsSynchronizer.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Workflows/WorkflowsSynchronizer.cs index 19c687ed..2667aa96 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Workflows/WorkflowsSynchronizer.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Workflows/WorkflowsSynchronizer.cs @@ -19,7 +19,7 @@ public sealed class WorkflowsSynchronizer : ISynchronizer private const string Ref = "../__json/workflow"; private readonly ILogger log; - public string Name => "Workflow"; + public string Name => "Workflows"; public WorkflowsSynchronizer(ILogger log) { diff --git a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj index 0d8d09ae..97c51c58 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj +++ b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj @@ -14,7 +14,7 @@ net5.0 true sq - 7.16 + 7.17 diff --git a/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Management/Custom.cs b/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Management/Custom.cs index f1b2b0dc..7cfbf568 100644 --- a/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Management/Custom.cs +++ b/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Management/Custom.cs @@ -19,9 +19,7 @@ namespace Squidex.ClientLibrary.Management { -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public partial class ErrorDto -#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member { /// public override string ToString() @@ -47,9 +45,7 @@ public override string ToString() } } -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public partial class SquidexManagementException -#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member { /// public override string ToString() @@ -148,9 +144,7 @@ internal string ToIdString() } } -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public partial interface IAssetsClient -#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member { /// /// Upload a new asset. @@ -200,11 +194,23 @@ public partial interface IAssetsClient /// /// A server side error occurred. Task GetAllAsync(string app, Func callback, int batchSize = 200, CancellationToken cancellationToken = default); + + /// + /// Get assets. + /// + /// The name of the app. + /// The callback that is invoke for each asset. + /// The optional asset query. + /// The number of assets per request. + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Assets returned. + /// + /// A server side error occurred. + Task GetAllByQueryAsync(string app, Func callback, AssetQuery query = null, int batchSize = 200, CancellationToken cancellationToken = default); } -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public partial class AssetsClient -#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member { /// public Task PostAssetAsync(string app, string parentId = null, string id = null, bool? duplicate = null, FileInfo file = null, CancellationToken cancellationToken = default) @@ -229,12 +235,25 @@ public Task GetAssetsAsync(string app, AssetQuery query = null, Cance } /// - public async Task GetAllAsync(string app, Func callback, int batchSize = 200, CancellationToken cancellationToken = default) + public Task GetAllAsync(string app, Func callback, int batchSize = 200, CancellationToken cancellationToken = default) + { + return GetAllByQueryAsync(app, callback, null, batchSize, cancellationToken); + } + + /// + public async Task GetAllByQueryAsync(string app, Func callback, AssetQuery query = null, int batchSize = 200, CancellationToken cancellationToken = default) { Guard.Between(batchSize, 10, 10_000, nameof(batchSize)); Guard.NotNull(callback, nameof(callback)); - var query = new AssetQuery { Top = batchSize, Skip = 0 }; + if (query == null) + { + query = new AssetQuery(); + } + + query.Top = batchSize; + query.Skip = 0; + var added = new HashSet(); do {