From e2f25f081ce692e18c29db2719e0f59a4204ac2f Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 28 May 2021 17:13:53 +0200 Subject: [PATCH] Raw mapping for json. --- .../Squidex.CLI.Tests/MapCSVToSquidexTests.cs | 41 +++++++++- .../MapJsonToSquidexTests.cs | 3 +- ...quidexTests.cs => MapSquidexToCSVTests.cs} | 11 +-- .../TestDataGeneratorTests.cs | 8 +- .../ImExport/Csv2SquidexConverter.cs | 11 ++- .../ImExport/Json2SquidexConverter.cs | 4 +- .../Implementation/ImExport/JsonMapping.cs | 81 +++++++++++++------ .../Squidex.CLI/Squidex.CLI.csproj | 2 +- .../Utils/HttpClientExtensions.cs | 27 ++++--- 9 files changed, 138 insertions(+), 50 deletions(-) rename cli/Squidex.CLI/Squidex.CLI.Tests/{MapFromSquidexTests.cs => MapSquidexToCSVTests.cs} (93%) diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/MapCSVToSquidexTests.cs b/cli/Squidex.CLI/Squidex.CLI.Tests/MapCSVToSquidexTests.cs index b4030348..d1896a42 100644 --- a/cli/Squidex.CLI/Squidex.CLI.Tests/MapCSVToSquidexTests.cs +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/MapCSVToSquidexTests.cs @@ -12,6 +12,7 @@ using CsvHelper; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Squidex.CLI.Commands.Implementation; using Squidex.CLI.Commands.Implementation.ImExport; using Squidex.ClientLibrary; using Xunit; @@ -54,13 +55,13 @@ public MapCSVToSquidexTests() [Fact] public void Should_throw_exception_if_field_names_is_null() { - Assert.Throws(() => new Csv2SquidexConverter(null)); + Assert.Throws(() => new Csv2SquidexConverter(null)); } [Fact] public void Should_throw_exception_if_field_names_is_empty() { - Assert.Throws(() => new Csv2SquidexConverter(string.Empty)); + Assert.Throws(() => new Csv2SquidexConverter(string.Empty)); } [Fact] @@ -117,6 +118,42 @@ public void Should_read_number_to_invariant() EqualJson(expected, actual); } + [Fact] + public void Should_read_number_string() + { + var sut = new Csv2SquidexConverter("number/raw"); + + var actual = sut.ReadAll(csvReader).First(); + + var expected = new DynamicData + { + ["number"] = new JObject + { + ["iv"] = "1234" + } + }; + + EqualJson(expected, actual); + } + + [Fact] + public void Should_read_number_from_path_to_string() + { + var sut = new Csv2SquidexConverter("path=number/raw"); + + var actual = sut.ReadAll(csvReader).First(); + + var expected = new DynamicData + { + ["path"] = new JObject + { + ["iv"] = "1234" + } + }; + + EqualJson(expected, actual); + } + [Fact] public void Should_read_string_to_localized() { diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/MapJsonToSquidexTests.cs b/cli/Squidex.CLI/Squidex.CLI.Tests/MapJsonToSquidexTests.cs index 96b3b4ec..74754ec5 100644 --- a/cli/Squidex.CLI/Squidex.CLI.Tests/MapJsonToSquidexTests.cs +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/MapJsonToSquidexTests.cs @@ -9,6 +9,7 @@ using System.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Squidex.CLI.Commands.Implementation; using Squidex.CLI.Commands.Implementation.ImExport; using Squidex.ClientLibrary; using Xunit; @@ -47,7 +48,7 @@ public void Should_not_throw_exception_if_field_names_is_null() [Fact] public void Should_throw_exception_if_field_names_is_empty() { - Assert.Throws(() => new Json2SquidexConverter(string.Empty)); + Assert.Throws(() => new Json2SquidexConverter(string.Empty)); } [Fact] diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/MapFromSquidexTests.cs b/cli/Squidex.CLI/Squidex.CLI.Tests/MapSquidexToCSVTests.cs similarity index 93% rename from cli/Squidex.CLI/Squidex.CLI.Tests/MapFromSquidexTests.cs rename to cli/Squidex.CLI/Squidex.CLI.Tests/MapSquidexToCSVTests.cs index 614e99e9..8b2342cb 100644 --- a/cli/Squidex.CLI/Squidex.CLI.Tests/MapFromSquidexTests.cs +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/MapSquidexToCSVTests.cs @@ -1,4 +1,4 @@ -// ========================================================================== +// ========================================================================== // Squidex Headless CMS // ========================================================================== // Copyright (c) Squidex UG (haftungsbeschraenkt) @@ -8,17 +8,18 @@ using System; using System.Linq; using Newtonsoft.Json.Linq; +using Squidex.CLI.Commands.Implementation; using Squidex.CLI.Commands.Implementation.ImExport; using Squidex.ClientLibrary; using Xunit; namespace Squidex.CLI.Tests { - public class MapFromSquidexTests + public class MapSquidexToCSVTests { private readonly DynamicContent content; - public MapFromSquidexTests() + public MapSquidexToCSVTests() { content = new DynamicContent { @@ -59,13 +60,13 @@ public MapFromSquidexTests() [Fact] public void Should_throw_exception_if_field_names_is_null() { - Assert.Throws(() => new Squidex2CsvConverter(null)); + Assert.Throws(() => new Squidex2CsvConverter(null)); } [Fact] public void Should_throw_exception_if_field_names_is_empty() { - Assert.Throws(() => new Squidex2CsvConverter(string.Empty)); + Assert.Throws(() => new Squidex2CsvConverter(string.Empty)); } [Fact] diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/TestDataGeneratorTests.cs b/cli/Squidex.CLI/Squidex.CLI.Tests/TestDataGeneratorTests.cs index a1e50b64..e2276836 100644 --- a/cli/Squidex.CLI/Squidex.CLI.Tests/TestDataGeneratorTests.cs +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/TestDataGeneratorTests.cs @@ -38,7 +38,7 @@ public void Should_generate_datetime_between_min_and_max() var max = dates.Max(); Assert.True(min >= minValue.Date); - Assert.True(max <= maxValue.Date); + Assert.True(max <= maxValue); Assert.True(dates.All(x => x.Kind == DateTimeKind.Utc)); } @@ -64,7 +64,7 @@ public void Should_generate_datetime_before_max() var max = dates.Max(); Assert.True(min >= minValue.Date); - Assert.True(max <= maxValue.Date); + Assert.True(max <= maxValue); Assert.True(dates.All(x => x.Kind == DateTimeKind.Utc)); } @@ -90,7 +90,7 @@ public void Should_generate_datetime_after_min() var max = dates.Max(); Assert.True(min >= minValue.Date); - Assert.True(max <= maxValue.Date); + Assert.True(max <= maxValue); Assert.True(dates.All(x => x.Kind == DateTimeKind.Utc)); } @@ -114,7 +114,7 @@ public void Should_generate_datetime_with_default_range() var max = dates.Max(); Assert.True(min >= minValue.Date); - Assert.True(max <= maxValue.Date); + Assert.True(max <= maxValue); Assert.True(dates.All(x => x.Kind == DateTimeKind.Utc)); } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Csv2SquidexConverter.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Csv2SquidexConverter.cs index f7f00adc..91772992 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Csv2SquidexConverter.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Csv2SquidexConverter.cs @@ -33,13 +33,20 @@ public IEnumerable ReadAll(CsvReader csvReader) { var data = new DynamicData(); - foreach (var (name, path) in mapping) + foreach (var (name, path, format) in mapping) { if (csvReader.TryGetField(name, out var value)) { try { - SetValue(data, JToken.Parse(value), path); + if (format == "raw") + { + SetValue(data, value, path); + } + else + { + SetValue(data, JToken.Parse(value), path); + } } catch (JsonReaderException) { diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Json2SquidexConverter.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Json2SquidexConverter.cs index 6cc07b84..5f8d8169 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Json2SquidexConverter.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/Json2SquidexConverter.cs @@ -1,4 +1,4 @@ -// ========================================================================== +// ========================================================================== // Squidex Headless CMS // ========================================================================== // Copyright (c) Squidex UG (haftungsbeschraenkt) @@ -51,7 +51,7 @@ public DynamicData ReadOne(JsonTextReader jsonReader) if (mapping != null) { - foreach (var (name, path) in mapping) + foreach (var (name, path, _) in mapping) { if (item.TryGetValue(name, out var value)) { diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/JsonMapping.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/JsonMapping.cs index 9dcb7617..ae5c81de 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/JsonMapping.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/ImExport/JsonMapping.cs @@ -7,20 +7,26 @@ using System; using System.Collections.Generic; +using System.Text.RegularExpressions; namespace Squidex.CLI.Commands.Implementation.ImExport { - public sealed class JsonMapping : List<(string Name, JsonPath Path)> + public sealed class JsonMapping : List<(string Name, JsonPath Path, string Format)> { + private static readonly Regex FormatRegex = new Regex("(?[^\\/=]*)(=(?[^\\/]*))?(\\/(?.*))?", RegexOptions.Compiled); + public static JsonMapping ForJson2Csv(string fields) { fields ??= string.Empty; var result = new JsonMapping(); - foreach (var item in fields.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + foreach (var field in fields.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) { - var parts = item.Split('='); + if (string.IsNullOrWhiteSpace(field)) + { + continue; + } static JsonPath GetPath(string value) { @@ -34,16 +40,23 @@ static JsonPath GetPath(string value) return path; } - switch (parts.Length) + var match = FormatRegex.Match(field); + + if (match.Success) { - case 1: - result.Add((parts[0], GetPath(parts[0]))); - break; - case 2: - result.Add((parts[0], GetPath(parts[1]))); - break; - default: - throw new CLIException("Field definition not valid."); + var name = match.Groups["Lhs"].Value; + var path = name; + + if (match.Groups["Rhs"].Success) + { + path = match.Groups["Rhs"].Value; + } + + result.Add((name, GetPath(path), GetFormat(match))); + } + else + { + throw new CLIException("Field definition not valid."); } } @@ -61,9 +74,12 @@ public static JsonMapping ForCsv2Json(string fields) var result = new JsonMapping(); - foreach (var item in fields.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + foreach (var field in fields.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) { - var parts = item.Split('='); + if (string.IsNullOrWhiteSpace(field)) + { + continue; + } static JsonPath GetPath(string value) { @@ -77,16 +93,23 @@ static JsonPath GetPath(string value) return path; } - switch (parts.Length) + var match = FormatRegex.Match(field); + + if (match.Success) { - case 1: - result.Add((parts[0], GetPath(parts[0]))); - break; - case 2: - result.Add((parts[1], GetPath(parts[0]))); - break; - default: - throw new CLIException("Field definition not valid."); + var name = match.Groups["Lhs"].Value; + var path = name; + + if (match.Groups["Rhs"].Success) + { + name = match.Groups["Rhs"].Value; + } + + result.Add((name, GetPath(path), GetFormat(match))); + } + else + { + throw new CLIException("Field definition not valid."); } } @@ -98,6 +121,18 @@ static JsonPath GetPath(string value) return result; } + private static string GetFormat(Match match) + { + var format = "json"; + + if (match.Groups["Format"].Success) + { + format = match.Groups["Format"].Value; + } + + return format; + } + private static bool IsDataDraft(JsonPath path) { return string.Equals(path[0].Key, "DataDraft", StringComparison.OrdinalIgnoreCase); diff --git a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj index 5b313e19..611241b9 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj +++ b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj @@ -14,7 +14,7 @@ net5.0 true sq - 7.6 + 7.7 diff --git a/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Utils/HttpClientExtensions.cs b/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Utils/HttpClientExtensions.cs index 629dff30..25132735 100644 --- a/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Utils/HttpClientExtensions.cs +++ b/csharp/Squidex.ClientLibrary/Squidex.ClientLibrary/Utils/HttpClientExtensions.cs @@ -20,19 +20,20 @@ namespace Squidex.ClientLibrary.Utils { public static class HttpClientExtensions { - private static readonly JsonSerializerSettings SerializerSettings = CreateSerializer(); + private static readonly JsonSerializerSettings SerializerSettings; + private static readonly JsonSerializer Serializer; - private static JsonSerializerSettings CreateSerializer() + static HttpClientExtensions() { - var result = new JsonSerializerSettings + SerializerSettings = new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver() }; - result.Converters.Add(new StringEnumConverter()); - result.Converters.Add(new UTCIsoDateTimeConverter()); + SerializerSettings.Converters.Add(new StringEnumConverter()); + SerializerSettings.Converters.Add(new UTCIsoDateTimeConverter()); - return result; + Serializer = JsonSerializer.CreateDefault(SerializerSettings); } public static HttpContent ToContent(this T value) @@ -67,10 +68,16 @@ public static string ToJson(this T value) public static async Task ReadAsJsonAsync(this HttpContent content) { - var jsonString = await content.ReadAsStringAsync(); - var jsonObject = JsonConvert.DeserializeObject(jsonString); - - return jsonObject; + using (var stream = await content.ReadAsStreamAsync()) + { + using (var reader = new StreamReader(stream)) + { + using (var jsonReader = new JsonTextReader(reader)) + { + return Serializer.Deserialize(jsonReader); + } + } + } } } }