From 5c4f8e5d87c440aa4ae473cd189bdd2beef49e71 Mon Sep 17 00:00:00 2001 From: wangkechun Date: Mon, 29 Jan 2024 16:12:05 +0800 Subject: [PATCH 1/4] add JSONToGoStruct --- src/core/config/Categories.json | 6 + src/core/lib/JSONToGoStruct.mjs | 457 +++++++++++++++++++++++++ src/core/operations/JSONToGoStruct.mjs | 66 ++++ 3 files changed, 529 insertions(+) create mode 100644 src/core/lib/JSONToGoStruct.mjs create mode 100644 src/core/operations/JSONToGoStruct.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index cf4d91be09..79664d3528 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -3,6 +3,12 @@ "name": "Favourites", "ops": [] }, + { + "name": "Wangkechun", + "ops": [ + "JSON To Go Struct" + ] + }, { "name": "Data format", "ops": [ diff --git a/src/core/lib/JSONToGoStruct.mjs b/src/core/lib/JSONToGoStruct.mjs new file mode 100644 index 0000000000..2a3de4a2d0 --- /dev/null +++ b/src/core/lib/JSONToGoStruct.mjs @@ -0,0 +1,457 @@ +/* + JSON-to-Go + by Matt Holt + + https://github.com/mholt/json-to-go + + A simple utility to translate JSON into a Go type definition. +*/ + +export function jsonToGo( + json, + typename, + flatten = true, + example = false, + allOmitempty = false +) { + let data; + let scope; + let go = ""; + let tabs = 0; + + const seen = {}; + const stack = []; + let accumulator = ""; + let innerTabs = 0; + let parent = ""; + + try { + data = JSON.parse(json.replace(/(:\s*\[?\s*-?\d*)\.0/g, "$1.1")); // hack that forces floats to stay as floats + scope = data; + } catch (e) { + return { + go: "", + error: e.message, + }; + } + + typename = format(typename || "AutoGenerated"); + append(`type ${typename} `); + + parseScope(scope); + + return { + go: flatten ? (go += accumulator) : go, + }; + + function parseScope(scope, depth = 0) { + if (typeof scope === "object" && scope !== null) { + if (Array.isArray(scope)) { + let sliceType; + const scopeLength = scope.length; + + for (let i = 0; i < scopeLength; i++) { + const thisType = goType(scope[i]); + if (!sliceType) sliceType = thisType; + else if (sliceType != thisType) { + sliceType = mostSpecificPossibleGoType( + thisType, + sliceType + ); + if (sliceType == "any") break; + } + } + + const slice = + flatten && ["struct", "slice"].includes(sliceType) + ? `[]${parent}` + : `[]`; + + if (flatten && depth >= 2) appender(slice); + else append(slice); + if (sliceType == "struct") { + const allFields = {}; + + // for each field counts how many times appears + for (let i = 0; i < scopeLength; i++) { + const keys = Object.keys(scope[i]); + for (let k in keys) { + let keyname = keys[k]; + if (!(keyname in allFields)) { + allFields[keyname] = { + value: scope[i][keyname], + count: 0, + }; + } else { + const existingValue = allFields[keyname].value; + const currentValue = scope[i][keyname]; + + if ( + compareObjects(existingValue, currentValue) + ) { + const comparisonResult = compareObjectKeys( + Object.keys(currentValue), + Object.keys(existingValue) + ); + if (!comparisonResult) { + keyname = `${keyname}_${uuidv4()}`; + allFields[keyname] = { + value: currentValue, + count: 0, + }; + } + } + } + allFields[keyname].count++; + } + } + + // create a common struct with all fields found in the current array + // omitempty dict indicates if a field is optional + const keys = Object.keys(allFields), + struct = {}, + omitempty = {}; + for (let k in keys) { + const keyname = keys[k], + elem = allFields[keyname]; + + struct[keyname] = elem.value; + omitempty[keyname] = elem.count != scopeLength; + } + parseStruct(depth + 1, innerTabs, struct, omitempty); // finally parse the struct !! + } else if (sliceType == "slice") { + parseScope(scope[0], depth); + } else { + if (flatten && depth >= 2) { + appender(sliceType || "any"); + } else { + append(sliceType || "any"); + } + } + } else { + if (flatten) { + if (depth >= 2) { + appender(parent); + } else { + append(parent); + } + } + parseStruct(depth + 1, innerTabs, scope); + } + } else { + if (flatten && depth >= 2) { + appender(goType(scope)); + } else { + append(goType(scope)); + } + } + } + + function parseStruct(depth, innerTabs, scope, omitempty) { + if (flatten) { + stack.push(depth >= 2 ? "\n" : ""); + } + + const seenTypeNames = []; + + if (flatten && depth >= 2) { + const parentType = `type ${parent}`; + const scopeKeys = formatScopeKeys(Object.keys(scope)); + + // this can only handle two duplicate items + // future improvement will handle the case where there could + // three or more duplicate keys with different values + if (parent in seen && compareObjectKeys(scopeKeys, seen[parent])) { + stack.pop(); + return; + } + seen[parent] = scopeKeys; + + appender(`${parentType} struct {\n`); + ++innerTabs; + const keys = Object.keys(scope); + for (let i in keys) { + const keyname = getOriginalName(keys[i]); + indenter(innerTabs); + const typename = uniqueTypeName(format(keyname), seenTypeNames); + seenTypeNames.push(typename); + + appender(typename + " "); + parent = typename; + parseScope(scope[keys[i]], depth); + appender(' `json:"' + keyname); + if ( + allOmitempty || + (omitempty && omitempty[keys[i]] === true) + ) { + appender(",omitempty"); + } + appender('"`\n'); + } + indenter(--innerTabs); + appender("}"); + } else { + append("struct {\n"); + ++tabs; + const keys = Object.keys(scope); + for (let i in keys) { + const keyname = getOriginalName(keys[i]); + indent(tabs); + const typename = uniqueTypeName(format(keyname), seenTypeNames); + seenTypeNames.push(typename); + append(typename + " "); + parent = typename; + parseScope(scope[keys[i]], depth); + append(' `json:"' + keyname); + if ( + allOmitempty || + (omitempty && omitempty[keys[i]] === true) + ) { + append(",omitempty"); + } + if ( + example && + scope[keys[i]] !== "" && + typeof scope[keys[i]] !== "object" + ) { + append('" example:"' + scope[keys[i]]); + } + append('"`\n'); + } + indent(--tabs); + append("}"); + } + if (flatten) accumulator += stack.pop(); + } + + function indent(tabs) { + for (let i = 0; i < tabs; i++) go += "\t"; + } + + function append(str) { + go += str; + } + + function indenter(tabs) { + for (let i = 0; i < tabs; i++) stack[stack.length - 1] += "\t"; + } + + function appender(str) { + stack[stack.length - 1] += str; + } + + // Generate a unique name to avoid duplicate struct field names. + // This function appends a number at the end of the field name. + function uniqueTypeName(name, seen) { + if (seen.indexOf(name) === -1) { + return name; + } + + let i = 0; + while (true) { + let newName = name + i.toString(); + if (seen.indexOf(newName) === -1) { + return newName; + } + + i++; + } + } + + // Sanitizes and formats a string to make an appropriate identifier in Go + function format(str) { + str = formatNumber(str); + + let sanitized = toProperCase(str).replace(/[^a-z0-9]/gi, ""); + if (!sanitized) { + return "NAMING_FAILED"; + } + + // After sanitizing the remaining characters can start with a number. + // Run the sanitized string again trough formatNumber to make sure the identifier is Num[0-9] or Zero_... instead of 1. + return formatNumber(sanitized); + } + + // Adds a prefix to a number to make an appropriate identifier in Go + function formatNumber(str) { + if (!str) return ""; + else if (str.match(/^\d+$/)) str = "Num" + str; + else if (str.charAt(0).match(/\d/)) { + const numbers = { + 0: "Zero_", + 1: "One_", + 2: "Two_", + 3: "Three_", + 4: "Four_", + 5: "Five_", + 6: "Six_", + 7: "Seven_", + 8: "Eight_", + 9: "Nine_", + }; + str = numbers[str.charAt(0)] + str.substr(1); + } + + return str; + } + + // Determines the most appropriate Go type + function goType(val) { + if (val === null) return "any"; + + switch (typeof val) { + case "string": + if ( + /\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(\+\d\d:\d\d|Z)/.test( + val + ) + ) + return "time.Time"; + else return "string"; + case "number": + if (val % 1 === 0) { + if (val > -2147483648 && val < 2147483647) return "int"; + else return "int64"; + } else return "float64"; + case "boolean": + return "bool"; + case "object": + if (Array.isArray(val)) return "slice"; + return "struct"; + default: + return "any"; + } + } + + // Given two types, returns the more specific of the two + function mostSpecificPossibleGoType(typ1, typ2) { + if (typ1.substr(0, 5) == "float" && typ2.substr(0, 3) == "int") + return typ1; + else if (typ1.substr(0, 3) == "int" && typ2.substr(0, 5) == "float") + return typ2; + else return "any"; + } + + // Proper cases a string according to Go conventions + function toProperCase(str) { + // ensure that the SCREAMING_SNAKE_CASE is converted to snake_case + if (str.match(/^[_A-Z0-9]+$/)) { + str = str.toLowerCase(); + } + + // https://github.com/golang/lint/blob/5614ed5bae6fb75893070bdc0996a68765fdd275/lint.go#L771-L810 + const commonInitialisms = [ + "ACL", + "API", + "ASCII", + "CPU", + "CSS", + "DNS", + "EOF", + "GUID", + "HTML", + "HTTP", + "HTTPS", + "ID", + "IP", + "JSON", + "LHS", + "QPS", + "RAM", + "RHS", + "RPC", + "SLA", + "SMTP", + "SQL", + "SSH", + "TCP", + "TLS", + "TTL", + "UDP", + "UI", + "UID", + "UUID", + "URI", + "URL", + "UTF8", + "VM", + "XML", + "XMPP", + "XSRF", + "XSS", + ]; + + return str + .replace(/(^|[^a-zA-Z])([a-z]+)/g, function (unused, sep, frag) { + if (commonInitialisms.indexOf(frag.toUpperCase()) >= 0) + return sep + frag.toUpperCase(); + else + return ( + sep + + frag[0].toUpperCase() + + frag.substr(1).toLowerCase() + ); + }) + .replace(/([A-Z])([a-z]+)/g, function (unused, sep, frag) { + if (commonInitialisms.indexOf(sep + frag.toUpperCase()) >= 0) + return (sep + frag).toUpperCase(); + else return sep + frag; + }); + } + + function uuidv4() { + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace( + /[xy]/g, + function (c) { + var r = (Math.random() * 16) | 0, + v = c == "x" ? r : (r & 0x3) | 0x8; + return v.toString(16); + } + ); + } + + function getOriginalName(unique) { + const reLiteralUUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + const uuidLength = 36; + + if (unique.length >= uuidLength) { + const tail = unique.substr(-uuidLength); + if (reLiteralUUID.test(tail)) { + return unique.slice(0, -1 * (uuidLength + 1)); + } + } + return unique; + } + + function compareObjects(objectA, objectB) { + const object = "[object Object]"; + return ( + Object.prototype.toString.call(objectA) === object && + Object.prototype.toString.call(objectB) === object + ); + } + + function compareObjectKeys(itemAKeys, itemBKeys) { + const lengthA = itemAKeys.length; + const lengthB = itemBKeys.length; + + // nothing to compare, probably identical + if (lengthA == 0 && lengthB == 0) return true; + + // duh + if (lengthA != lengthB) return false; + + for (let item of itemAKeys) { + if (!itemBKeys.includes(item)) return false; + } + return true; + } + + function formatScopeKeys(keys) { + for (let i in keys) { + keys[i] = format(keys[i]); + } + return keys; + } +} diff --git a/src/core/operations/JSONToGoStruct.mjs b/src/core/operations/JSONToGoStruct.mjs new file mode 100644 index 0000000000..4f2e4ddde2 --- /dev/null +++ b/src/core/operations/JSONToGoStruct.mjs @@ -0,0 +1,66 @@ +/** + * @author wangkechun [hi@hi-hi.cn] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import { jsonToGo } from "../lib/JSONToGoStruct.mjs"; +import JSON5 from "json5"; +import OperationError from "../errors/OperationError.mjs"; + +/** + * JSON To Go Struct operation + */ +class JSONToGoStruct extends Operation { + /** + * JSONToGoStruct constructor + */ + constructor() { + super(); + + this.name = "JSON To Go Struct"; + this.module = "Default"; + this.description = "converts JSON into a Go type definition."; + this.infoURL = "https://mholt.github.io/json-to-go/"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Type Name", + type: "string", + value: "AutoGenerated", + }, + { + name: "Flatten", + type: "boolean", + value: false, + }, + { + name: "All Omit Empty", + type: "boolean", + value: false, + }, + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [typename, flatten, allOmitempty] = args; + if (!input) return ""; + let code; + try { + code = JSON.stringify(JSON5.parse(input)); + } catch (err) { + throw new OperationError("Unable to parse input as JSON.\n" + err); + } + const result = jsonToGo(code, typename, flatten, false, allOmitempty); + return result["go"]; + } +} + +export default JSONToGoStruct; From 7f2355b782d3091a5899de1143b37cc0c8b3c1a0 Mon Sep 17 00:00:00 2001 From: wangkechun Date: Mon, 29 Jan 2024 17:28:50 +0800 Subject: [PATCH 2/4] test pass --- src/core/config/Categories.json | 7 +-- src/core/operations/JSONToGoStruct.mjs | 10 ++-- src/core/{lib => vendor}/JSONToGoStruct.mjs | 9 +++ tests/operations/index.mjs | 1 + tests/operations/tests/JSONToGoStruct.mjs | 62 +++++++++++++++++++++ 5 files changed, 78 insertions(+), 11 deletions(-) rename src/core/{lib => vendor}/JSONToGoStruct.mjs (97%) create mode 100644 tests/operations/tests/JSONToGoStruct.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 79664d3528..90406c5c3e 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -3,12 +3,6 @@ "name": "Favourites", "ops": [] }, - { - "name": "Wangkechun", - "ops": [ - "JSON To Go Struct" - ] - }, { "name": "Data format", "ops": [ @@ -420,6 +414,7 @@ "JavaScript Minify", "JSON Beautify", "JSON Minify", + "JSON to Go Struct", "XML Beautify", "XML Minify", "SQL Beautify", diff --git a/src/core/operations/JSONToGoStruct.mjs b/src/core/operations/JSONToGoStruct.mjs index 4f2e4ddde2..fce2b665d1 100644 --- a/src/core/operations/JSONToGoStruct.mjs +++ b/src/core/operations/JSONToGoStruct.mjs @@ -5,12 +5,12 @@ */ import Operation from "../Operation.mjs"; -import { jsonToGo } from "../lib/JSONToGoStruct.mjs"; +import { jsonToGo } from "../vendor/JSONToGoStruct.mjs"; import JSON5 from "json5"; import OperationError from "../errors/OperationError.mjs"; /** - * JSON To Go Struct operation + * JSON to Go Struct operation */ class JSONToGoStruct extends Operation { /** @@ -19,7 +19,7 @@ class JSONToGoStruct extends Operation { constructor() { super(); - this.name = "JSON To Go Struct"; + this.name = "JSON to Go Struct"; this.module = "Default"; this.description = "converts JSON into a Go type definition."; this.infoURL = "https://mholt.github.io/json-to-go/"; @@ -34,7 +34,7 @@ class JSONToGoStruct extends Operation { { name: "Flatten", type: "boolean", - value: false, + value: true, }, { name: "All Omit Empty", @@ -59,7 +59,7 @@ class JSONToGoStruct extends Operation { throw new OperationError("Unable to parse input as JSON.\n" + err); } const result = jsonToGo(code, typename, flatten, false, allOmitempty); - return result["go"]; + return result.go; } } diff --git a/src/core/lib/JSONToGoStruct.mjs b/src/core/vendor/JSONToGoStruct.mjs similarity index 97% rename from src/core/lib/JSONToGoStruct.mjs rename to src/core/vendor/JSONToGoStruct.mjs index 2a3de4a2d0..ff38907b7a 100644 --- a/src/core/lib/JSONToGoStruct.mjs +++ b/src/core/vendor/JSONToGoStruct.mjs @@ -171,6 +171,9 @@ export function jsonToGo( ++innerTabs; const keys = Object.keys(scope); for (let i in keys) { + if (!Object.prototype.hasOwnProperty.call(keys, i)) { + continue; + } const keyname = getOriginalName(keys[i]); indenter(innerTabs); const typename = uniqueTypeName(format(keyname), seenTypeNames); @@ -195,6 +198,9 @@ export function jsonToGo( ++tabs; const keys = Object.keys(scope); for (let i in keys) { + if (!Object.prototype.hasOwnProperty.call(keys, i)) { + continue; + } const keyname = getOriginalName(keys[i]); indent(tabs); const typename = uniqueTypeName(format(keyname), seenTypeNames); @@ -450,6 +456,9 @@ export function jsonToGo( function formatScopeKeys(keys) { for (let i in keys) { + if (!Object.prototype.hasOwnProperty.call(keys, i)) { + continue; + } keys[i] = format(keys[i]); } return keys; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 570fbb6fe4..25a4f125d9 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -59,6 +59,7 @@ import "./tests/Jump.mjs"; import "./tests/JSONBeautify.mjs"; import "./tests/JSONMinify.mjs"; import "./tests/JSONtoCSV.mjs"; +import "./tests/JSONToGoStruct.mjs"; import "./tests/JWTDecode.mjs"; import "./tests/JWTSign.mjs"; import "./tests/JWTVerify.mjs"; diff --git a/tests/operations/tests/JSONToGoStruct.mjs b/tests/operations/tests/JSONToGoStruct.mjs new file mode 100644 index 0000000000..8b8f2f0868 --- /dev/null +++ b/tests/operations/tests/JSONToGoStruct.mjs @@ -0,0 +1,62 @@ +/** + * JSON to Go Struct tests. + * + * @author wangkechun [hi@hi-hi.cn] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "JSON to Go Struct: simple", + input: JSON.stringify({ a: "1", b: "2", c: "3" }), + expectedOutput: `type AutoGenerated struct { + A string \`json:"a"\` + B string \`json:"b"\` + C string \`json:"c"\` +}`.replaceAll(" ", "\t"), + recipeConfig: [ + { + op: "JSON to Go Struct", + args: ["AutoGenerated", true, false], + }, + ], + }, + { + name: "JSON to Go Struct: flatten", + input: JSON.stringify({ a: "1", b: "2", c: { d: "e" } }), + expectedOutput: `type AutoGenerated struct { + A string \`json:"a"\` + B string \`json:"b"\` + C C \`json:"c"\` +} +type C struct { + D string \`json:"d"\` +}`.replaceAll(" ", "\t"), + recipeConfig: [ + { + op: "JSON to Go Struct", + args: ["AutoGenerated", true, false], + }, + ], + }, + { + name: "JSON to Go Struct: nest", + input: JSON.stringify({ a: "1", b: "2", c: { d: "e" } }), + expectedOutput: `type AutoGenerated struct { + A string \`json:"a"\` + B string \`json:"b"\` + C struct { + D string \`json:"d"\` + } \`json:"c"\` +}`.replaceAll(" ", "\t"), + recipeConfig: [ + { + op: "JSON to Go Struct", + args: ["AutoGenerated", false, false], + }, + ], + }, +]); From e9430daad4dcee0c9e14054da84653e739832c77 Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:33:56 +0000 Subject: [PATCH 3/4] Change category, capitalise description --- src/core/config/Categories.json | 4 ++-- src/core/operations/JSONToGoStruct.mjs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 90406c5c3e..32d5e7255b 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -67,7 +67,8 @@ "JSON to CSV", "Avro to JSON", "CBOR Encode", - "CBOR Decode" + "CBOR Decode", + "JSON to Go Struct" ] }, { @@ -414,7 +415,6 @@ "JavaScript Minify", "JSON Beautify", "JSON Minify", - "JSON to Go Struct", "XML Beautify", "XML Minify", "SQL Beautify", diff --git a/src/core/operations/JSONToGoStruct.mjs b/src/core/operations/JSONToGoStruct.mjs index fce2b665d1..08bfa587f5 100644 --- a/src/core/operations/JSONToGoStruct.mjs +++ b/src/core/operations/JSONToGoStruct.mjs @@ -21,7 +21,7 @@ class JSONToGoStruct extends Operation { this.name = "JSON to Go Struct"; this.module = "Default"; - this.description = "converts JSON into a Go type definition."; + this.description = "Converts JSON into a Go type definition."; this.infoURL = "https://mholt.github.io/json-to-go/"; this.inputType = "string"; this.outputType = "string"; From dc2bb0f8cce556d98721c95de690bcb3ce5dd4c2 Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:40:42 +0000 Subject: [PATCH 4/4] Add comma in categories --- src/core/config/Categories.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 7718be9847..a484177597 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -70,7 +70,7 @@ "Avro to JSON", "CBOR Encode", "CBOR Decode", - "JSON to Go Struct" + "JSON to Go Struct", "Rison Encode", "Rison Decode" ]