Skip to content

Commit

Permalink
New feature to generate contents.
Browse files Browse the repository at this point in the history
  • Loading branch information
SebastianStehle committed Jun 24, 2023
1 parent 2b0a308 commit 7c449d7
Show file tree
Hide file tree
Showing 41 changed files with 698 additions and 188 deletions.
11 changes: 8 additions & 3 deletions .editorconfig → cli/Squidex.CLI/.editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ insert_final_newline = true
indent_style = space
indent_size = 4

csharp_style_namespace_declarations = file_scoped

# FAILING ANALYZERS
dotnet_diagnostic.RECS0002.severity = none
dotnet_diagnostic.RECS0117.severity = none
Expand All @@ -22,6 +24,9 @@ dotnet_diagnostic.SA1649.severity = none
# CA1707: Identifiers should not contain underscores
dotnet_diagnostic.CA1707.severity = none

# CA2016: Forward the 'CancellationToken' parameter to methods
dotnet_diagnostic.CA2016.severity = none

# CS8618: Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
dotnet_diagnostic.CS8618.severity = none

Expand Down Expand Up @@ -81,9 +86,6 @@ dotnet_diagnostic.MA0038.severity = none

# MA0039: Do not write your own certificate validation method
dotnet_diagnostic.MA0039.severity = none

# MA0048: File name must match type name
dotnet_diagnostic.MA0048.severity = none

# MA0049: Type name should not match containing namespace
dotnet_diagnostic.MA0049.severity = none
Expand Down Expand Up @@ -166,5 +168,8 @@ dotnet_diagnostic.SA1601.severity = none
# SA1602: Enumeration items should be documented
dotnet_diagnostic.SA1602.severity = none

# SA1615: Element return value should be documented
dotnet_diagnostic.SA1615.severity = none

# SA1623: Property summary documentation should match accessors
dotnet_diagnostic.SA1623.severity = none
2 changes: 1 addition & 1 deletion cli/Squidex.CLI/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@
<PackageTags>Squidex HeadlessCMS</PackageTags>
<PublishRepositoryUrl>true</PublishRepositoryUrl>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
<Version>10.3</Version>
<Version>11.0</Version>
</PropertyGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System.Text;
using Markdig;
using Markdig.Syntax;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using OpenAI;
using OpenAI.Managers;
using OpenAI.ObjectModels.RequestModels;
using OpenAI.ObjectModels.ResponseModels;
using Squidex.CLI.Commands.Implementation.Utils;
using Squidex.CLI.Configuration;
using Squidex.ClientLibrary;

namespace Squidex.CLI.Commands.Implementation.AI;

public sealed class AIContentGenerator
{
private readonly IConfigurationStore configurationStore;

public AIContentGenerator(IConfigurationStore configurationStore)
{
this.configurationStore = configurationStore;
}

public async Task<GeneratedContent> GenerateAsync(string description, string apiKey, string? schemaName = null,
CancellationToken ct = default)
{
var cachedResponse = await MakeRequestAsync(description, apiKey, ct);

return ParseResult(schemaName, cachedResponse);
}

private async Task<ChatCompletionCreateResponse> MakeRequestAsync(string description, string apiKey,
CancellationToken ct)
{
var client = new OpenAIService(new OpenAiOptions
{
ApiKey = apiKey,
});

var cacheKey = $"openapi/query-cache/{description.ToSha256Base64()}";
var (cachedResponse, _) = configurationStore.Get<ChatCompletionCreateResponse>(cacheKey);

if (cachedResponse == null)
{
cachedResponse = await client.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
{
ChatMessage.FromSystem("Create a list as json array. The list is described as followed:"),
ChatMessage.FromUser(description),
ChatMessage.FromSystem("Also create a JSON object with the field names of this list as keys and the json type as value (string)."),
ChatMessage.FromSystem("Also create a JSON array that only contains the name of the list above as valid slug."),
},
Model = OpenAI.ObjectModels.Models.ChatGpt3_5Turbo
}, cancellationToken: ct);

configurationStore.Set(cacheKey, cachedResponse);
}

return cachedResponse;
}

private static GeneratedContent ParseResult(string? schemaName, ChatCompletionCreateResponse cachedResponse)
{
var parsed = Markdown.Parse(cachedResponse.Choices[0].Message.Content);

var codeBlocks = parsed.OfType<FencedCodeBlock>().ToList();
if (codeBlocks.Count != 3)
{
ThrowParsingException("3 code blocks expected");
return default!;
}

var schemaObject = ParseJson<JObject>(codeBlocks[1], "Schema");
var schemaFields = new List<UpsertSchemaFieldDto>();

foreach (var (key, value) in schemaObject)
{
var fieldType = value?.ToString();

switch (fieldType?.ToLowerInvariant())
{
case "string":
case "text":
schemaFields.Add(new UpsertSchemaFieldDto
{
Name = key!,
Properties = new StringFieldPropertiesDto()
});
break;
case "double":
case "float":
case "int":
case "integer":
case "long":
case "number":
case "real":
schemaFields.Add(new UpsertSchemaFieldDto
{
Name = key!,
Properties = new NumberFieldPropertiesDto()
});
break;
case "bit":
case "bool":
case "boolean":
schemaFields.Add(new UpsertSchemaFieldDto
{
Name = key!,
Properties = new BooleanFieldPropertiesDto()
});
break;
default:
ThrowParsingException($"Unexpected field type '{fieldType}' for field '{key}'");
return default!;
}
}

var nameArray = ParseJson<JArray>(codeBlocks[2], "SchemaName");
if (nameArray.Count != 1)
{
ThrowParsingException("'SchemaName' json has an unexpected structure.");
return default!;
}

if (string.IsNullOrWhiteSpace(schemaName))
{
schemaName = nameArray[0].ToString();
}

var contentsBlock = ParseJson<JArray>(codeBlocks[0], "Contents");
var contentsList = new List<Dictionary<string, object>>();

foreach (var obj in contentsBlock.OfType<JObject>())
{
contentsList.Add(obj.OfType<JProperty>().ToDictionary(x => x.Name, x => (object)x.Value));
}

return new GeneratedContent
{
SchemaFields = schemaFields,
SchemaName = schemaName,
Contents = contentsList
};
}

private static void ThrowParsingException(string reason)
{
throw new InvalidOperationException($"OpenAPI does not return a parsable result: {reason}.");
}

private static T ParseJson<T>(LeafBlock block, string name) where T : JToken
{
JToken jsonNode;
try
{
var jsonText = GetText(block);

jsonNode = JToken.Parse(jsonText);
}
catch (JsonException)
{
ThrowParsingException($"'{name}' code is not valid json.");
return default!;
}

if (jsonNode is not T typed)
{
ThrowParsingException($"'{name}' json has an unexpected structure.");
return default!;
}

return typed;

static string GetText(LeafBlock block)
{
var sb = new StringBuilder();

var lines = block.Lines.Lines;

if (lines != null)
{
foreach (var line in lines)
{
sb.AppendLine(line.Slice.ToString());
}
}

return sb.ToString();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using Squidex.ClientLibrary;

namespace Squidex.CLI.Commands.Implementation.AI;

public sealed class GeneratedContent
{
public List<UpsertSchemaFieldDto> SchemaFields { get; set; }

public string SchemaName { get; set; }

public List<Dictionary<string, object>> Contents { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,6 @@ public static async Task ExportAsync(this ISession session, IExportSettings sett
while (totalRead < total);
}

log.WriteLine("> Exported: {0} of {1}. Completed.", totalRead, total);
log.Completed($"Export of {totalRead}/{total} content items completed.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public static async Task ImportAsync(this ISession session, IImportSettings sett
}
}

log.WriteLine("> Imported: {0}. Completed.", totalWritten);
log.Completed($"Import of {totalWritten} content items completed");
}

public static IEnumerable<DynamicData> Read(this Csv2SquidexConverter converter, Stream stream, string delimiter)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,11 @@ public static void StepFailed(this ILogger log, Exception ex)
HandleException(ex, log.StepFailed);
}

public static void Completed(this ILogger log, string message)
{
log.WriteLine($"> {message}");
}

public static void HandleException(Exception ex, Action<string> error)
{
switch (ex)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs)
var (asset, path) = item;

var process = $"Downloading {path}";

try
{
var assetFile = fs.GetFile(path);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -105,6 +106,34 @@ public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> source, i
}
}

public static string WithoutPrefix(this string value, string prefix)
{
if (value.EndsWith(prefix, StringComparison.OrdinalIgnoreCase))
{
return value[..^prefix.Length];
}

return value;
}

public static string ToSha256Base64(this string value)
{
return ToSha256Base64(Encoding.UTF8.GetBytes(value));
}

public static string ToSha256Base64(this byte[] bytes)
{
var hashBytes = SHA256.HashData(bytes);

var result =
Convert.ToBase64String(hashBytes)
.Replace("+", string.Empty, StringComparison.Ordinal)
.Replace("=", string.Empty, StringComparison.Ordinal)
.ToLowerInvariant();

return result;
}

public static DirectoryInfo CreateDirectory(this DirectoryInfo directory, string name)
{
return Directory.CreateDirectory(Path.Combine(directory.FullName, name));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,3 @@
// ==========================================================================

namespace Squidex.CLI.Configuration;

public sealed class Configuration
{
public Dictionary<string, ConfiguredApp> Apps { get; } = new Dictionary<string, ConfiguredApp>();

public string? CurrentApp { get; set; }
}
Loading

0 comments on commit 7c449d7

Please sign in to comment.