Skip to content

Commit

Permalink
Handle and resolve references before making the upsert. (#44)
Browse files Browse the repository at this point in the history 
* Handle and resolve references before making the upsert.

* Fix for reference resolves.

* Simplifying queries

* Logging improved.

* Fix topological sort.
  • Loading branch information
@SebastianStehle
SebastianStehle authored May 5, 2020
1 parent 1466966 commit 9846ebe
Show file tree
Hide file tree
Showing 7 changed files with 401 additions and 65 deletions.
34 changes: 27 additions & 7 deletions cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Extension.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static async Task DoVersionedAsync(this ILogger log, string process, long
}
catch (Exception ex)
{
HandleException(ex, log);
HandleException(ex, log.StepFailed);
}
}

Expand All @@ -73,35 +73,55 @@ public static async Task DoSafeAsync(this ILogger log, string process, Func<Task
}
catch (Exception ex)
{
HandleException(ex, log);
HandleException(ex, log.StepFailed);
}
}

private static void HandleException(Exception ex, ILogger log)
public static async Task DoSafeLineAsync(this ILogger log, string process, Func<Task> action)
{
try
{
log.WriteLine("Start: {0}", process);

await action();

log.WriteLine("Done : {0}", process);
}
catch (Exception ex)
{
HandleException(ex, error => log.WriteLine("Error: {0}; {1}", process, error));
}
finally
{
log.WriteLine();
}
}

private static void HandleException(Exception ex, Action<string> error)
{
switch (ex)
{
case SquidexManagementException<ErrorDto> ex1:
{
log.StepFailed(ex1.Result.ToString());
error(ex1.Result.ToString());
break;
}

case SquidexManagementException ex2:
{
log.StepFailed(ex2.Message);
error(ex2.Message);
break;
}

case CLIException ex4:
{
log.StepSkipped(ex4.Message);
error(ex4.Message);
break;
}

case Exception ex3:
{
log.StepFailed(ex3);
error(ex3.ToString());
throw ex3;
}
}
Expand Down
203 changes: 203 additions & 0 deletions cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentGroup.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using Squidex.ClientLibrary;

namespace Squidex.CLI.Commands.Implementation.Sync.Contents
{
public sealed class ContentGroup : IEquatable<ContentGroup>
{
public string SchemaName { get; }

public List<ContentModel> Contents { get; }

public HashSet<string> Dependencies { get; } = new HashSet<string>();

public ContentGroup(string schemaName, IEnumerable<ContentModel> contents)
{
SchemaName = schemaName;

Contents = contents.ToList();

foreach (var content in contents)
{
if (content.References != null)
{
foreach (var reference in content.References.Values)
{
Dependencies.Add(reference.Schema);
}
}
}
}

public async Task ResolveReferencesAsync(ISession session, ILogger log, ReferenceCache cache)
{
foreach (var content in Contents.ToList())
{
if (content.References?.Any() == true)
{
await ResolveContentAsync(content, session, log, cache);
}
}
}

public async Task UpsertAsync(ISession session, ILogger log, ReferenceCache cache)
{
var client = session.Contents(SchemaName);

var request = new BulkUpdate
{
Jobs = Contents.Select(x => new BulkUpdateJob { Query = new { filter = x.Filter }, Data = x.Data }).ToList()
};

var results = await client.BulkUpdateAsync(request);

var i = 0;

foreach (var result in results)
{
var content = Contents[i];

log.StepStart($"{content.File} / {content.Ref}");

if (result.ContentId != null)
{
log.StepSuccess();
}
else if (result.Error != null)
{
log.StepFailed(result.Error.ToString());
}
else
{
log.StepSkipped("Unknown Reason");
}

i++;
}
}

private async Task ResolveContentAsync(ContentModel content, ISession session, ILogger log, ReferenceCache cache)
{
var resolvedReferences = new Dictionary<string, Guid>();

foreach (var (key, value) in content.References.ToList())
{
var schema = value.Schema;

var filter = JObject.FromObject(value.Filter);
var filterJson = filter.ToString();

if (cache.TryGetValue((schema, filterJson), out var id))
{
resolvedReferences[key] = id;
continue;
}

var client = session.Contents(value.Schema);

var query = new { filter, take = 1 };

var references = await client.GetAsync(new ContentQuery { JsonQuery = query }, QueryContext.Default.Unpublished(true));

if (references.Total == 1)
{
id = references.Items[0].Id;

resolvedReferences[key] = id;

cache[(schema, filterJson)] = id;
}
else if (references.Total > 1)
{
log.WriteLine("Skipping {0} / {1} reference {2} not unique.", content.File, content.Ref, key);

Contents.Remove(content);
return;
}
else
{
log.WriteLine("Skipping {0} / {1} reference {2} not resolved.", content.File, content.Ref, key);

Contents.Remove(content);
return;
}
}

UpdateData(content.Data, resolvedReferences);
}

private void UpdateData(JToken data, Dictionary<string, Guid> resolvedReferences)
{
if (data is JArray array)
{
for (var i = 0; i < array.Count; i++)
{
var value = array[i];

if (value.Type == JTokenType.String)
{
if (resolvedReferences.TryGetValue(value.Value<string>(), out var id))
{
array[i] = id;
}
}
else
{
UpdateData(value, resolvedReferences);
}
}
}
else if (data is JObject obj)
{
ICollection<KeyValuePair<string, JToken>> list = obj;

foreach (var (key, value) in list.ToList())
{
if (value.Type == JTokenType.String)
{
if (resolvedReferences.TryGetValue(value.Value<string>(), out var id))
{
data[key] = id;
}
}
else
{
UpdateData(value, resolvedReferences);
}
}
}
}

public override bool Equals(object obj)
{
return Equals(obj as ContentGroup);
}

public bool Equals(ContentGroup other)
{
return string.Equals(other?.SchemaName, SchemaName);
}

public override int GetHashCode()
{
return SchemaName.GetHashCode();
}

public override string ToString()
{
return SchemaName;
}
}
}
18 changes: 16 additions & 2 deletions cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NJsonSchema.Annotations;

namespace Squidex.CLI.Commands.Implementation.Sync.Contents
{
Expand All @@ -15,8 +19,18 @@ public sealed class ContentModel
public string Schema { get; set; }

[Required]
public object Data { get; set; }
public JObject Data { get; set; }

public object Query { get; set; }
public JObject Filter { get; set; }

public Dictionary<string, ContentReference> References { get; set; }

[JsonSchemaIgnore]
[JsonIgnore]
public string File { get; set; }

[JsonSchemaIgnore]
[JsonIgnore]
public string Ref { get; set; }
}
}
20 changes: 20 additions & 0 deletions cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentReference.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System.ComponentModel.DataAnnotations;

namespace Squidex.CLI.Commands.Implementation.Sync.Contents
{
public sealed class ContentReference
{
[Required]
public string Schema { get; set; }

[Required]
public object Filter { get; set; }
}
}
Loading

0 comments on commit 9846ebe

Please sign in to comment.