refactor templatize ev2 package

tooling/templatize/pkg/config/config.go

@@ -247,20 +247,28 @@ func (cp *configProviderImpl) loadConfig(configReplacements *ConfigReplacements)
 // PreprocessFile reads and processes a gotemplate
 // The path will be read as is. It parses the file as a template, and executes it with the provided variables.
 func PreprocessFile(templateFilePath string, vars map[string]any) ([]byte, error) {
-	tmpl := template.New("file")
 	content, err := os.ReadFile(templateFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file %s: %w", templateFilePath, err)
 	}
+	processedContent, err := PreprocessContent(content, vars)
+	if err != nil {
+		return nil, fmt.Errorf("failed to preprocess file %s: %w", templateFilePath, err)
+	}
+	return processedContent, nil
+}
 
-	tmpl, err = tmpl.Parse(string(content))
+// PreprocessContent processes a gotemplate from memory
+func PreprocessContent(content []byte, vars map[string]any) ([]byte, error) {
+	tmpl := template.New("file")
+	tmpl, err := tmpl.Parse(string(content))
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse template %s: %w", templateFilePath, err)
+		return nil, fmt.Errorf("failed to parse template: %w", err)
 	}
 
 	var tmplBytes bytes.Buffer
 	if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil {
-		return nil, fmt.Errorf("failed to execute template %s: %w", templateFilePath, err)
+		return nil, fmt.Errorf("failed to execute template: %w", err)
 	}
 	return tmplBytes.Bytes(), nil
 }

tooling/templatize/pkg/config/config_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil"
 )
 
 func TestConfigProvider(t *testing.T) {
@@ -231,3 +233,78 @@ func TestConvertToInterface(t *testing.T) {
 	assert.IsType(t, expected, map[string]any{})
 	assert.IsType(t, expected["key2"], map[string]any{})
 }
+
+func TestPreprocessContent(t *testing.T) {
+	fileContent, err := os.ReadFile("../../testdata/test.bicepparam")
+	assert.Nil(t, err)
+
+	processed, err := PreprocessContent(
+		fileContent,
+		map[string]any{
+			"regionRG": "bahamas",
+			"clusterService": map[string]any{
+				"imageTag": "cs-image",
+			},
+		},
+	)
+	assert.Nil(t, err)
+	testutil.CompareWithFixture(t, processed, testutil.WithExtension(".bicepparam"))
+}
+
+func TestPreprocessContentMissingKey(t *testing.T) {
+	testCases := []struct {
+		name       string
+		content    string
+		vars       map[string]any
+		shouldFail bool
+	}{
+		{
+			name:    "missing key",
+			content: "foo: {{ .bar }}",
+			vars: map[string]any{
+				"baz": "bar",
+			},
+			shouldFail: true,
+		},
+		{
+			name:    "missing nested key",
+			content: "foo: {{ .bar.baz }}",
+			vars: map[string]any{
+				"baz": "bar",
+			},
+			shouldFail: true,
+		},
+		{
+			name:    "no missing key",
+			content: "foo: {{ .bar }}",
+			vars: map[string]any{
+				"bar": "bar",
+			},
+			shouldFail: false,
+		},
+		{
+			name:    "no missing nested key",
+			content: "foo: {{ .bar.baz }}",
+			vars: map[string]any{
+				"bar": map[string]any{
+					"baz": "baz",
+				},
+			},
+			shouldFail: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := PreprocessContent(
+				[]byte(tc.content),
+				tc.vars,
+			)
+			if tc.shouldFail {
+				assert.NotNil(t, err)
+			} else {
+				assert.Nil(t, err)
+			}
+		})
+	}
+}

tooling/templatize/pkg/ev2/pipeline.go

@@ -1,6 +1,7 @@
 package ev2
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -10,73 +11,105 @@ import (
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
-func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (string, error) {
-	// switch to the pipeline file dir so all relative paths are resolved correctly
-	originalDir, err := os.Getwd()
-	if err != nil {
-		return "", nil
-	}
-	pipelineDir := filepath.Dir(pipelineFilePath)
-	err = os.Chdir(pipelineDir)
+const precompiledPrefix = "ev2-precompiled-"
+
+func PrecompilePipelineFileForEV2(pipelineFilePath string, vars config.Variables) (string, error) {
+	precompiledPipeline, err := PrecompilePipelineForEV2(pipelineFilePath, vars)
 	if err != nil {
 		return "", err
 	}
-	defer func() {
-		_ = os.Chdir(originalDir)
-	}()
 
-	// precompile the pipeline file
-	pipelineFileName := filepath.Base(pipelineFilePath)
-	p, err := pipeline.NewPipelineFromFile(pipelineFileName, vars)
+	// store as new file
+	pipelineBytes, err := yaml.Marshal(precompiledPipeline)
 	if err != nil {
 		return "", err
 	}
-	err = processPipelineForEV2(p, vars)
+	err = os.WriteFile(precompiledPipeline.PipelineFilePath(), pipelineBytes, 0644)
 	if err != nil {
 		return "", err
 	}
 
-	// store as new file
-	pipelineBytes, err := yaml.Marshal(p)
+	return precompiledPipeline.PipelineFilePath(), nil
+}
+
+func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (*pipeline.Pipeline, error) {
+	// load the pipeline and referenced files
+	originalPipeline, err := pipeline.NewPipelineFromFile(pipelineFilePath, vars)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	newPipelineFileName := "ev2-precompiled-" + pipelineFileName
-	err = os.WriteFile(newPipelineFileName, pipelineBytes, 0644)
+	referencedFiles, err := readReferencedPipelineFiles(originalPipeline)
 	if err != nil {
-		return "", err
+		return nil, fmt.Errorf("failed to read referenced files of pipeline %s: %w", originalPipeline.PipelineFilePath(), err)
+	}
+
+	// precompile the pipeline and referenced files
+	processedPipeline, processedFiles, err := processPipelineForEV2(originalPipeline, referencedFiles, vars)
+	if err != nil {
+		return nil, err
+	}
+
+	// store the processed files to disk relative to the pipeline directory
+	for filePath, content := range processedFiles {
+		absFilePath, err := processedPipeline.AbsoluteFilePath(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get absolute file path for %q: %w", filePath, err)
+		}
+		err = os.WriteFile(absFilePath, content, 0644)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write precompiled file %q: %w", filePath, err)
+		}
 	}
 
-	return filepath.Join(pipelineDir, newPipelineFileName), nil
+	return processedPipeline, nil
 }
 
-func processPipelineForEV2(p *pipeline.Pipeline, vars config.Variables) error {
-	_, scopeBoundVars := EV2Mapping(vars, []string{})
+func readReferencedPipelineFiles(p *pipeline.Pipeline) (map[string][]byte, error) {
+	referencedFiles := make(map[string][]byte)
 	for _, rg := range p.ResourceGroups {
 		for _, step := range rg.Steps {
 			if step.Parameters != "" {
-				newParameterFilePath, err := precompileFileAndStore(step.Parameters, scopeBoundVars)
+				absFilePath, err := p.AbsoluteFilePath(step.Parameters)
 				if err != nil {
-					return err
+					return nil, fmt.Errorf("failed to get absolute file path for %q: %w", step.Parameters, err)
 				}
-				step.Parameters = newParameterFilePath
+				paramFileContent, err := os.ReadFile(absFilePath)
+				if err != nil {
+					return nil, fmt.Errorf("failed to read parameter file %q: %w", step.Parameters, err)
+				}
+				referencedFiles[step.Parameters] = paramFileContent
 			}
 		}
 	}
-	return nil
+	return referencedFiles, nil
 }
 
-func precompileFileAndStore(filePath string, vars map[string]interface{}) (string, error) {
-	preprocessedBytes, err := config.PreprocessFile(filePath, vars)
+func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]byte, vars config.Variables) (*pipeline.Pipeline, map[string][]byte, error) {
+	processingPipeline, err := p.DeepCopy(buildPrefixedFilePath(p.PipelineFilePath(), precompiledPrefix))
 	if err != nil {
-		return "", err
+		return nil, nil, err
 	}
-	newFilePath := buildPrefixedFilePath(filePath, "ev2-precompiled-")
-	err = os.WriteFile(newFilePath, preprocessedBytes, 0644)
-	if err != nil {
-		return "", err
+	processedFiles := make(map[string][]byte)
+	_, scopeBoundVars := EV2Mapping(vars, []string{})
+	for _, rg := range processingPipeline.ResourceGroups {
+		for _, step := range rg.Steps {
+			// preprocess the parameters file with scopebinding variables
+			if step.Parameters != "" {
+				paramFileContent, ok := referencedFiles[step.Parameters]
+				if !ok {
+					return nil, nil, fmt.Errorf("parameter file %q not found", step.Parameters)
+				}
+				preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundVars)
+				if err != nil {
+					return nil, nil, err
+				}
+				newParameterFilePath := buildPrefixedFilePath(step.Parameters, precompiledPrefix)
+				processedFiles[newParameterFilePath] = preprocessedBytes
+				step.Parameters = newParameterFilePath
+			}
+		}
 	}
-	return newFilePath, nil
+	return processingPipeline, processedFiles, nil
 }
 
 func buildPrefixedFilePath(path, prefix string) string {

tooling/templatize/pkg/ev2/pipeline_test.go

@@ -1,40 +1,42 @@
 package ev2
 
 import (
-	"fmt"
-	"os"
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
-func TestPrecompilePipelineForEV2(t *testing.T) {
-	defer func() {
-		_ = os.Remove("../../testdata/ev2-precompiled-pipeline.yaml")
-		_ = os.Remove("../../testdata/ev2-precompiled-test.bicepparam")
-	}()
-
+func TestProcessPipelineForEV2(t *testing.T) {
 	configProvider := config.NewConfigProvider("../../testdata/config.yaml")
-	vars, err := configProvider.GetVariables("public", "int", "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables("public", "int", "", NewEv2ConfigReplacements())
 	if err != nil {
 		t.Errorf("failed to get variables: %v", err)
 	}
-	newPipelinePath, err := PrecompilePipelineForEV2("../../testdata/pipeline.yaml", vars)
+	originalPipeline, err := pipeline.NewPipelineFromFile("../../testdata/pipeline.yaml", vars)
+	if err != nil {
+		t.Errorf("failed to read new pipeline: %v", err)
+	}
+	files := make(map[string][]byte)
+	files["test.bicepparam"] = []byte("param regionRG = '{{ .regionRG }}'")
+
+	newPipeline, newFiles, err := processPipelineForEV2(originalPipeline, files, vars)
 	if err != nil {
 		t.Errorf("failed to precompile pipeline: %v", err)
 	}
 
-	p, err := pipeline.NewPipelineFromFile(newPipelinePath, vars)
+	// verify pipeline
+	pipelineContent, err := yaml.Marshal(newPipeline)
 	if err != nil {
-		t.Errorf("failed to read new pipeline: %v", err)
+		t.Errorf("failed to marshal processed pipeline: %v", err)
 	}
-	fmt.Println(p)
-	expectedParamsPath := "ev2-precompiled-test.bicepparam"
+	testutil.CompareWithFixture(t, pipelineContent, testutil.WithExtension("pipeline.yaml"))
 
-	armStep := p.ResourceGroups[0].Steps[2]
-	if armStep.Parameters != expectedParamsPath {
-		t.Errorf("expected parameters path %v, but got %v", expectedParamsPath, armStep.Parameters)
+	// verify referenced files
+	for filePath, content := range newFiles {
+		testutil.CompareWithFixture(t, content, testutil.WithExtension(filePath))
 	}
-	// TODO improve test, check against fixture
 }

tooling/templatize/pkg/ev2/utils.go

@@ -10,7 +10,7 @@ import (
 // This package contains helper functions to extract EV2 conformant data from a config.yaml file.
 //
 
-func newEv2ConfigReplacements() *config.ConfigReplacements {
+func NewEv2ConfigReplacements() *config.ConfigReplacements {
 	return config.NewConfigReplacements(
 		"$location()",
 		"$(regionShortName)",
@@ -23,7 +23,7 @@ func newEv2ConfigReplacements() *config.ConfigReplacements {
 // The variable values are formatted to contain EV2 $location(), $stamp() and $(serviceConfigVar) variables.
 // This function is useful to get the variables to fill the `Settings` section of an EV2 `ServiceConfig.json“
 func GetNonRegionalServiceConfigVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (config.Variables, error) {
-	return configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	return configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 }
 
 // GetRegionalServiceConfigVariableOverrides returns the regional overrides of a config.yaml file.
@@ -36,7 +36,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi
 	}
 	overrides := make(map[string]config.Variables)
 	for _, region := range regions {
-		regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, newEv2ConfigReplacements())
+		regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, NewEv2ConfigReplacements())
 		if err != nil {
 			return nil, err
 		}
@@ -49,7 +49,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi
 // It uses the provided configProvider to fetch the variables, flattens them into a __VAR__ = $config(var) formatted map.
 // This function is useful to get the find/replace pairs for an EV2 `ScopeBinding.json`
 func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (map[string]string, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +65,7 @@ func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEn
 // while maintaining EV2 conformant system variables.
 // This function is useful to process a pipeline.yaml file so that it contains EV2 system variables.
 func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +77,7 @@ func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud,
 // This function is useful to process bicepparam files so that they can be used within EV2 together
 // with scopebinding.
 func PreprocessFileForEV2ScopeBinding(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}