introduce step action based structs (#1005)

* introduce step action based structs Signed-off-by: Gerd Oberlechner <[email protected]> * remove parsing indirection on step level Signed-off-by: Gerd Oberlechner <[email protected]> * remove RG parsing magic func Signed-off-by: Gerd Oberlechner <[email protected]> * new types Signed-off-by: Gerd Oberlechner <[email protected]> * update testdata for configref Signed-off-by: Gerd Oberlechner <[email protected]> * json array handling Signed-off-by: Gerd Oberlechner <[email protected]> --------- Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Dec 19, 2024 · aee24b6 · aee24b6
1 parent 40af6ce
commit aee24b6
Show file tree

Hide file tree

Showing 27 changed files with 644 additions and 340 deletions.
diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore
@@ -15,3 +15,5 @@ configurations/global-acr.bicepparam
 configurations/global-roles.bicepparam
 configurations/global-infra.bicepparam
 config.mk
+
+istio-*
diff --git a/tooling/templatize/cmd/generate/options.go b/tooling/templatize/cmd/generate/options.go
@@ -6,12 +6,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"text/template"
 
-	"github.com/Masterminds/sprig/v3"
 	"github.com/spf13/cobra"
 
 	options "github.com/Azure/ARO-HCP/tooling/templatize/cmd"
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
 
 func DefaultGenerationOptions() *RawGenerationOptions {
@@ -113,16 +112,9 @@ func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) {
 }
 
 func (opts *GenerationOptions) ExecuteTemplate() error {
-	tmpl := template.New(opts.InputFile).Funcs(sprig.FuncMap())
 	content, err := fs.ReadFile(opts.InputFS, opts.InputFile)
 	if err != nil {
 		return err
 	}
-
-	tmpl, err = tmpl.Parse(string(content))
-	if err != nil {
-		return err
-	}
-
-	return tmpl.Option("missingkey=error").ExecuteTemplate(opts.OutputFile, opts.InputFile, opts.RolloutOptions.Config)
+	return config.PreprocessContentIntoWriter(content, opts.RolloutOptions.Config, opts.OutputFile)
 }
diff --git a/tooling/templatize/cmd/pipeline/run/options.go b/tooling/templatize/cmd/pipeline/run/options.go
@@ -100,7 +100,7 @@ func (o *RunOptions) RunPipeline(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return o.PipelineOptions.Pipeline.Run(ctx, &pipeline.PipelineRunOptions{
+	return pipeline.RunPipeline(o.PipelineOptions.Pipeline, ctx, &pipeline.PipelineRunOptions{
 		DryRun:                o.DryRun,
 		Vars:                  variables,
 		Region:                rolloutOptions.Region,

diff --git a/tooling/templatize/internal/end2end/e2e.go b/tooling/templatize/internal/end2end/e2e.go
@@ -146,7 +146,7 @@ func (e *e2eImpl) SetAKSName(aksName string) {
 }
 
 func (e *e2eImpl) AddStep(step pipeline.Step, rg int) {
-	e.pipeline.ResourceGroups[rg].Steps = append(e.pipeline.ResourceGroups[rg].Steps, &step)
+	e.pipeline.ResourceGroups[rg].Steps = append(e.pipeline.ResourceGroups[rg].Steps, step)
 }
 
 func (e *e2eImpl) SetConfig(updates config.Variables) {

diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go
@@ -35,17 +35,13 @@ func TestE2EMake(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "test",
-		Action:  "Shell",
-		Command: "make test",
-		Variables: []pipeline.Variable{
-			{
-				Name:      "TEST_ENV",
-				ConfigRef: "test_env",
-			},
-		},
-	}, 0)
+	e2eImpl.AddStep(
+		pipeline.NewShellStep("test", "make test").WithVariables(pipeline.Variable{
+			Name:      "TEST_ENV",
+			ConfigRef: "test_env",
+		}),
+		0,
+	)
 
 	e2eImpl.SetConfig(config.Variables{"defaults": config.Variables{"test_env": "test_env"}})
 
@@ -68,11 +64,7 @@ func TestE2EKubernetes(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "test",
-		Action:  "Shell",
-		Command: "kubectl get namespaces",
-	}, 0)
+	e2eImpl.AddStep(pipeline.NewShellStep("test", "kubectl get namespaces"), 0)
 	e2eImpl.SetAKSName("aro-hcp-aks")
 
 	e2eImpl.SetConfig(config.Variables{"defaults": config.Variables{"rg": "hcp-underlay-dev-svc"}})
@@ -88,13 +80,7 @@ func TestE2EArmDeploy(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "test",
-		Action:     "ARM",
-		Template:   "test.bicep",
-		Parameters: "test.bicepparm",
-	}, 0)
-
+	e2eImpl.AddStep(pipeline.NewARMStep("test", "test.bicep", "test.bicepparm"), 0)
 	cleanup := e2eImpl.UseRandomRG()
 	defer func() {
 		err := cleanup()
@@ -140,11 +126,10 @@ func TestE2EShell(t *testing.T) {
 
 	e2eImpl := newE2E(tmpDir)
 
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "/bin/echo ${PWD} > env.txt",
-	}, 0)
+	e2eImpl.AddStep(
+		pipeline.NewShellStep("readInput", "/bin/echo ${PWD} > env.txt"),
+		0,
+	)
 
 	persistAndRun(t, &e2eImpl)
 
@@ -161,27 +146,20 @@ func TestE2EArmDeployWithOutput(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "createZone",
-		Action:     "ARM",
-		Template:   "test.bicep",
-		Parameters: "test.bicepparm",
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${zoneName} > env.txt",
-		Variables: []pipeline.Variable{
-			{
+
+	e2eImpl.AddStep(pipeline.NewARMStep("createZone", "test.bicep", "test.bicepparm"), 0)
+
+	e2eImpl.AddStep(pipeline.NewShellStep(
+		"readInput", "echo ${zoneName} > env.txt",
+	).WithVariables(
+		pipeline.Variable{
+			Name: "zoneName",
+			Input: &pipeline.Input{
 				Name: "zoneName",
-				Input: &pipeline.Input{
-					Name: "zoneName",
-					Step: "createZone",
-				},
+				Step: "createZone",
 			},
 		},
-	}, 0)
+	), 0)
 
 	cleanup := e2eImpl.UseRandomRG()
 	defer func() {
@@ -212,43 +190,26 @@ func TestE2EArmDeployWithOutputToArm(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterA",
-		Action:     "ARM",
-		Template:   "testa.bicep",
-		Parameters: "testa.bicepparm",
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterB",
-		Action:     "ARM",
-		Template:   "testb.bicep",
-		Parameters: "testb.bicepparm",
-		Variables: []pipeline.Variable{
-			{
-				Name: "parameterB",
-				Input: &pipeline.Input{
-					Name: "parameterA",
-					Step: "parameterA",
-				},
-			},
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterA", "testa.bicep", "testa.bicepparm"), 0)
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterB", "testb.bicep", "testb.bicepparm").WithVariables(pipeline.Variable{
+		Name: "parameterB",
+		Input: &pipeline.Input{
+			Name: "parameterA",
+			Step: "parameterA",
 		},
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${end} > env.txt",
-		Variables: []pipeline.Variable{
-			{
-				Name: "end",
-				Input: &pipeline.Input{
-					Name: "parameterC",
-					Step: "parameterB",
-				},
+	}), 0)
+
+	e2eImpl.AddStep(pipeline.NewShellStep(
+		"readInput", "echo ${end} > env.txt",
+	).WithVariables(
+		pipeline.Variable{
+			Name: "end",
+			Input: &pipeline.Input{
+				Name: "parameterC",
+				Step: "parameterB",
 			},
 		},
-	}, 0)
+	), 0)
 
 	e2eImpl.AddBicepTemplate(`
 param parameterA string
@@ -290,29 +251,19 @@ func TestE2EArmDeployWithOutputRGOverlap(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterA",
-		Action:     "ARM",
-		Template:   "testa.bicep",
-		Parameters: "testa.bicepparm",
-	}, 0)
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterA", "testa.bicep", "testa.bicepparm"), 0)
 
 	e2eImpl.AddResourceGroup()
 
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${end} > env.txt",
-		Variables: []pipeline.Variable{
-			{
-				Name: "end",
-				Input: &pipeline.Input{
-					Name: "parameterA",
-					Step: "parameterA",
-				},
+	e2eImpl.AddStep(pipeline.NewShellStep("readInput", "echo ${end} > env.txt").WithVariables(
+		pipeline.Variable{
+			Name: "end",
+			Input: &pipeline.Input{
+				Name: "parameterA",
+				Step: "parameterA",
 			},
 		},
-	}, 1)
+	), 1)
 
 	e2eImpl.AddBicepTemplate(`
 param parameterA string

diff --git a/tooling/templatize/pkg/config/config.go b/tooling/templatize/pkg/config/config.go
@@ -2,7 +2,9 @@ package config
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -260,15 +262,36 @@ func PreprocessFile(templateFilePath string, vars map[string]any) ([]byte, error
 
 // PreprocessContent processes a gotemplate from memory
 func PreprocessContent(content []byte, vars map[string]any) ([]byte, error) {
-	tmpl := template.New("file")
-	tmpl, err := tmpl.Parse(string(content))
+	var tmplBytes bytes.Buffer
+	if err := PreprocessContentIntoWriter(content, vars, &tmplBytes); err != nil {
+		return nil, err
+	}
+	return tmplBytes.Bytes(), nil
+}
+
+func PreprocessContentIntoWriter(content []byte, vars map[string]any, writer io.Writer) error {
+	funcMap := template.FuncMap{
+		"json": jsonEncoder,
+	}
+	tmpl, err := template.New("file").Funcs(funcMap).Parse(string(content))
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse template: %w", err)
+		return fmt.Errorf("failed to parse template: %w", err)
 	}
 
-	var tmplBytes bytes.Buffer
-	if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil {
-		return nil, fmt.Errorf("failed to execute template: %w", err)
+	if err := tmpl.Option("missingkey=error").Execute(writer, vars); err != nil {
+		return fmt.Errorf("failed to execute template: %w", err)
 	}
-	return tmplBytes.Bytes(), nil
+	return nil
+}
+
+func jsonEncoder(value interface{}) (string, error) {
+	valueType := reflect.TypeOf(value)
+	if valueType.Kind() == reflect.String {
+		return value.(string), nil
+	}
+	jsonBytes, err := json.Marshal(value)
+	if err != nil {
+		return "", err
+	}
+	return string(jsonBytes), nil
 }
diff --git a/tooling/templatize/pkg/config/config_test.go b/tooling/templatize/pkg/config/config_test.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil"
@@ -308,3 +309,48 @@ func TestPreprocessContentMissingKey(t *testing.T) {
 		})
 	}
 }
+
+func TestPreprocessContentJson(t *testing.T) {
+	templateContent := "{{ .variable | json }}"
+
+	testCases := []struct {
+		name           string
+		value          any
+		expectedResult string
+	}{
+		{
+			name:           "string value",
+			value:          "foo",
+			expectedResult: "foo",
+		},
+		{
+			name:           "array value",
+			value:          []string{"foo", "bar"},
+			expectedResult: "[\"foo\",\"bar\"]",
+		},
+		{
+			name:           "int value",
+			value:          42,
+			expectedResult: "42",
+		},
+		{
+			name:           "bool value",
+			value:          true,
+			expectedResult: "true",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			processed, err := PreprocessContent(
+				[]byte(templateContent),
+				map[string]any{
+					"variable": tc.value,
+				},
+			)
+			assert.Nil(t, err)
+			if diff := cmp.Diff(string(processed), string(tc.expectedResult)); diff != "" {
+				t.Errorf("got diff between expected and actual result\ndiff:\n%s\n\nIf this is expected, re-run the test with `UPDATE=true go test ./...` to update the fixtures.", diff)
+			}
+		})
+	}
+}
diff --git a/tooling/templatize/pkg/ev2/mapping_test.go b/tooling/templatize/pkg/ev2/mapping_test.go
@@ -128,6 +128,14 @@ func TestPlaceholderGenerators(t *testing.T) {
 			expectedFlattened: "__foo.bar__",
 			expectedReplace:   "any('__foo.bar__')",
 		},
+		{
+			name:              "bicep array param",
+			generator:         NewBicepParamPlaceholders(),
+			key:               []string{"foo", "bar"},
+			valueType:         reflect.TypeOf([]any{}),
+			expectedFlattened: "__foo.bar__",
+			expectedReplace:   "any('__foo.bar__')",
+		},
 	}
 
 	for _, tc := range tests {