adapt ev2 bicepparam preprocessor to generate any()

tooling/templatize/cmd/generate/options.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/cobra"
 
 	options "github.com/Azure/ARO-HCP/tooling/templatize/cmd"
-	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/ev2"
 )
 
 func DefaultGenerationOptions() *RawGenerationOptions {
@@ -28,7 +27,6 @@ func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error
 	}
 	cmd.Flags().StringVar(&opts.Input, "input", opts.Input, "input file path")
 	cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file path")
-	cmd.Flags().BoolVar(&opts.EV2Placeholders, "ev2-placeholders", opts.EV2Placeholders, "generate EV2 placeholders")
 
 	for _, flag := range []string{"config-file", "input", "output"} {
 		if err := cmd.MarkFlagFilename(flag); err != nil {
@@ -40,10 +38,9 @@ func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error
 
 // RawGenerationOptions holds input values.
 type RawGenerationOptions struct {
-	RolloutOptions  *options.RawRolloutOptions
-	Input           string
-	Output          string
-	EV2Placeholders bool
+	RolloutOptions *options.RawRolloutOptions
+	Input          string
+	Output         string
 }
 
 // validatedGenerationOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked.
@@ -94,11 +91,6 @@ func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) {
 		return nil, err
 	}
 
-	if o.EV2Placeholders {
-		_, vars := ev2.EV2Mapping(completed.Config, []string{})
-		completed.Config = vars
-	}
-
 	inputFile := filepath.Base(o.Input)
 
 	if err := os.MkdirAll(filepath.Dir(o.Output), os.ModePerm); err != nil {

tooling/templatize/pkg/ev2/mapping.go

@@ -7,23 +7,65 @@ import (
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
 
-func EV2Mapping(input config.Variables, prefix []string) (map[string]string, map[string]interface{}) {
+type PlaceholderGenerator func(key []string, value interface{}) (flattenedKey string, replaceVar string)
+
+// NewDunderPlaceholders returns a PlaceholderGenerator function that generates
+// placeholder strings by joining the provided key elements with underscores
+// and surrounding them with double underscores.
+//
+// The returned function takes a key (a slice of strings) and an unused interface{} value,
+// and returns a flattenedKey and replaceVar, both of which are the generated placeholder string.
+//
+// Example:
+//
+//	key := []string{"foo", "bar"}
+//	flattenedKey, replaceVar := NewDunderPlaceholders()(key, nil)
+//	// flattenedKey and replaceVar will both be "__foo_bar__"
+func NewDunderPlaceholders() PlaceholderGenerator {
+	return func(key []string, _ interface{}) (flattenedKey string, replaceVar string) {
+		flattenedKey = fmt.Sprintf("__%s__", strings.Join(key, "_"))
+		replaceVar = flattenedKey
+		return
+	}
+}
+
+// NewBicepParamPlaceholders creates a new PlaceholderGenerator that generates
+// placeholders for Bicep parameters. It uses DunderPlaceholders to generate
+// the initial placeholders and then wraps non-string values with the "any()"
+// function for general EV2 bicep happyness.
+//
+// Returns:
+//
+//	A PlaceholderGenerator function that takes a key and value, and returns
+//	a flattened key and a replacement variable for bicep parameter usage within EV2.
+func NewBicepParamPlaceholders() PlaceholderGenerator {
+	dunder := NewDunderPlaceholders()
+	return func(key []string, value interface{}) (flattenedKey string, replaceVar string) {
+		flattenedKey, replaceVar = dunder(key, value)
+		if _, ok := value.(string); !ok {
+			replaceVar = fmt.Sprintf("any('%s')", replaceVar)
+		}
+		return
+	}
+}
+
+func EV2Mapping(input config.Variables, placeholderGenerator PlaceholderGenerator, prefix []string) (map[string]string, map[string]interface{}) {
 	vars, _ := config.InterfaceToVariables(input)
 	output := map[string]string{}
 	replaced := map[string]interface{}{}
 	for key, value := range vars {
 		nestedKey := append(prefix, key)
 		nested, ok := value.(config.Variables)
 		if ok {
-			flattened, replacement := EV2Mapping(nested, nestedKey)
+			flattened, replacement := EV2Mapping(nested, placeholderGenerator, nestedKey)
 			for index, what := range flattened {
 				output[index] = what
 			}
 			replaced[key] = replacement
 		} else {
-			placeholder := fmt.Sprintf("__%s__", strings.Join(nestedKey, "_"))
-			output[placeholder] = strings.Join(nestedKey, ".")
-			replaced[key] = placeholder
+			flattenedKey, replaceVar := placeholderGenerator(nestedKey, value)
+			output[flattenedKey] = strings.Join(nestedKey, ".")
+			replaced[key] = replaceVar
 		}
 	}
 	return output, replaced

tooling/templatize/pkg/ev2/mapping_test.go

@@ -14,33 +14,74 @@ func TestMapping(t *testing.T) {
 		"key2": 42,
 		"key3": true,
 		"parent": map[string]interface{}{
-			"nested": "nestedvalue",
+			"nested":    "nestedvalue",
+			"nestedInt": 42,
 			"deeper": map[string]interface{}{
 				"deepest": "deepestvalue",
 			},
 		},
 	}
-	expectedFlattened := map[string]string{
-		"__key1__":                  "key1",
-		"__key2__":                  "key2",
-		"__key3__":                  "key3",
-		"__parent_nested__":         "parent.nested",
-		"__parent_deeper_deepest__": "parent.deeper.deepest",
-	}
-	expectedReplace := map[string]interface{}{
-		"key1": "__key1__",
-		"key2": "__key2__",
-		"key3": "__key3__",
-		"parent": map[string]interface{}{
-			"nested": "__parent_nested__",
-			"deeper": map[string]interface{}{"deepest": "__parent_deeper_deepest__"},
+	tests := []struct {
+		name              string
+		generator         PlaceholderGenerator
+		expectedFlattened map[string]string
+		expectedReplace   map[string]interface{}
+	}{
+		{
+			name:      "dunder",
+			generator: NewDunderPlaceholders(),
+			expectedFlattened: map[string]string{
+				"__key1__":                  "key1",
+				"__key2__":                  "key2",
+				"__key3__":                  "key3",
+				"__parent_nested__":         "parent.nested",
+				"__parent_nestedInt__":      "parent.nestedInt",
+				"__parent_deeper_deepest__": "parent.deeper.deepest",
+			},
+			expectedReplace: map[string]interface{}{
+				"key1": "__key1__",
+				"key2": "__key2__",
+				"key3": "__key3__",
+				"parent": map[string]interface{}{
+					"nested":    "__parent_nested__",
+					"nestedInt": "__parent_nestedInt__",
+					"deeper":    map[string]interface{}{"deepest": "__parent_deeper_deepest__"},
+				},
+			},
+		},
+		{
+			name:      "bicep",
+			generator: NewBicepParamPlaceholders(),
+			expectedFlattened: map[string]string{
+				"__key1__":                  "key1",
+				"__key2__":                  "key2",
+				"__key3__":                  "key3",
+				"__parent_nested__":         "parent.nested",
+				"__parent_nestedInt__":      "parent.nestedInt",
+				"__parent_deeper_deepest__": "parent.deeper.deepest",
+			},
+			expectedReplace: map[string]interface{}{
+				"key1": "__key1__",
+				"key2": "any('__key2__')",
+				"key3": "any('__key3__')",
+				"parent": map[string]interface{}{
+					"nested":    "__parent_nested__",
+					"nestedInt": "any('__parent_nestedInt__')",
+					"deeper":    map[string]interface{}{"deepest": "__parent_deeper_deepest__"},
+				},
+			},
 		},
 	}
-	flattened, replace := EV2Mapping(testData, []string{})
-	if diff := cmp.Diff(expectedFlattened, flattened); diff != "" {
-		t.Errorf("got incorrect flattened: %v", diff)
-	}
-	if diff := cmp.Diff(expectedReplace, replace); diff != "" {
-		t.Errorf("got incorrect replace: %v", diff)
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			flattened, replace := EV2Mapping(testData, tc.generator, []string{})
+			if diff := cmp.Diff(tc.expectedFlattened, flattened); diff != "" {
+				t.Errorf("got incorrect flattened: %v", diff)
+			}
+			if diff := cmp.Diff(tc.expectedReplace, replace); diff != "" {
+				t.Errorf("got incorrect replace: %v", diff)
+			}
+		})
 	}
 }

tooling/templatize/pkg/ev2/pipeline.go

@@ -90,7 +90,7 @@ func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]by
 		return nil, nil, err
 	}
 	processedFiles := make(map[string][]byte)
-	_, scopeBoundVars := EV2Mapping(vars, []string{})
+	_, scopeBoundBicepParamVars := EV2Mapping(vars, NewBicepParamPlaceholders(), []string{})
 	for _, rg := range processingPipeline.ResourceGroups {
 		for _, step := range rg.Steps {
 			// preprocess the parameters file with scopebinding variables
@@ -99,7 +99,7 @@ func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]by
 				if !ok {
 					return nil, nil, fmt.Errorf("parameter file %q not found", step.Parameters)
 				}
-				preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundVars)
+				preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundBicepParamVars)
 				if err != nil {
 					return nil, nil, err
 				}

tooling/templatize/pkg/ev2/pipeline_test.go

@@ -1,6 +1,7 @@
 package ev2
 
 import (
+	"strings"
 	"testing"
 
 	"gopkg.in/yaml.v3"
@@ -21,7 +22,15 @@ func TestProcessPipelineForEV2(t *testing.T) {
 		t.Errorf("failed to read new pipeline: %v", err)
 	}
 	files := make(map[string][]byte)
-	files["test.bicepparam"] = []byte("param regionRG = '{{ .regionRG }}'")
+	files["test.bicepparam"] = []byte(
+		strings.Join(
+			[]string{
+				"param regionRG = '{{ .regionRG }}'",
+				"param replicas = {{ .clusterService.replicas }}",
+			},
+			"\n",
+		),
+	)
 
 	newPipeline, newFiles, err := processPipelineForEV2(originalPipeline, files, vars)
 	if err != nil {

tooling/templatize/pkg/ev2/utils.go

@@ -53,34 +53,10 @@ func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEn
 	if err != nil {
 		return nil, err
 	}
-	flattened, _ := EV2Mapping(vars, []string{})
+	flattened, _ := EV2Mapping(vars, NewDunderPlaceholders(), []string{})
 	variables := make(map[string]string)
 	for key, value := range flattened {
 		variables[key] = fmt.Sprintf("$config(%s)", value)
 	}
 	return variables, nil
 }
-
-// PreprocessFileForEV2SystemVars processes an arbitrary gotemplate file and replaces all config.yaml references
-// while maintaining EV2 conformant system variables.
-// This function is useful to process a pipeline.yaml file so that it contains EV2 system variables.
-func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
-	if err != nil {
-		return nil, err
-	}
-	return config.PreprocessFile(templateFile, vars)
-}
-
-// PreprocessFileForEV2ScopeBinding processes an arbitrary gotemplate file and replaces all config.yaml references
-// with __VAR__ scope binding find/replace references.
-// This function is useful to process bicepparam files so that they can be used within EV2 together
-// with scopebinding.
-func PreprocessFileForEV2ScopeBinding(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
-	if err != nil {
-		return nil, err
-	}
-	_, scopeBindedVars := EV2Mapping(vars, []string{})
-	return config.PreprocessFile(templateFile, scopeBindedVars)
-}

tooling/templatize/pkg/ev2/utils_test.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 
-	"github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
 
@@ -28,27 +27,10 @@ func TestScopeBindingVariables(t *testing.T) {
 		"__serviceClusterRG__":              "$config(serviceClusterRG)",
 		"__serviceClusterSubscription__":    "$config(serviceClusterSubscription)",
 		"__clusterService_imageTag__":       "$config(clusterService.imageTag)",
+		"__clusterService_replicas__":       "$config(clusterService.replicas)",
 	}
 
 	if diff := cmp.Diff(expectedVars, vars); diff != "" {
 		t.Errorf("got incorrect vars: %v", diff)
 	}
 }
-
-func TestPreprocessFileForEV2SystemVars(t *testing.T) {
-	configProvider := config.NewConfigProvider("../../testdata/config.yaml")
-	content, err := PreprocessFileForEV2SystemVars(configProvider, "public", "int", "../../testdata/pipeline.yaml")
-	if err != nil {
-		t.Fatalf("PreprocessFileForEV2SystemVars failed: %v", err)
-	}
-	testutil.CompareWithFixture(t, content, testutil.WithExtension(".yaml"))
-}
-
-func TestPreprocessFileForEV2ScopeBinding(t *testing.T) {
-	configProvider := config.NewConfigProvider("../../testdata/config.yaml")
-	content, err := PreprocessFileForEV2ScopeBinding(configProvider, "public", "int", "../../testdata/test.bicepparam")
-	if err != nil {
-		t.Fatalf("PreprocessFileForEV2ScopeBinding failed: %v", err)
-	}
-	testutil.CompareWithFixture(t, content, testutil.WithExtension(".bicepparam"))
-}

tooling/templatize/testdata/config.yaml

@@ -11,6 +11,7 @@ defaults:
   aksName: aro-hcp-aks
   clusterService:
     imageTag: abcdef
+    replicas: 3
 clouds:
   fairfax:
     defaults:

tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2ev2-precompiled-test.bicepparam

@@ -1 +1,2 @@
-param regionRG = '__regionRG__'
+param regionRG = '__regionRG__'
+param replicas = any('__clusterService_replicas__')