introduce step action based structs

Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Dec 18, 2024 · e1c0d47 · e1c0d47
1 parent 71c65f1
commit e1c0d47
Show file tree

Hide file tree

Showing 16 changed files with 336 additions and 313 deletions.
diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore
@@ -15,3 +15,5 @@ configurations/global-acr.bicepparam
 configurations/global-roles.bicepparam
 configurations/global-infra.bicepparam
 config.mk
+
+istio-*
diff --git a/tooling/templatize/cmd/pipeline/run/options.go b/tooling/templatize/cmd/pipeline/run/options.go
@@ -100,7 +100,7 @@ func (o *RunOptions) RunPipeline(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return o.PipelineOptions.Pipeline.Run(ctx, &pipeline.PipelineRunOptions{
+	return pipeline.RunPipeline(o.PipelineOptions.Pipeline, ctx, &pipeline.PipelineRunOptions{
 		DryRun:                o.DryRun,
 		Vars:                  variables,
 		Region:                rolloutOptions.Region,

diff --git a/tooling/templatize/internal/end2end/e2e.go b/tooling/templatize/internal/end2end/e2e.go
@@ -146,7 +146,7 @@ func (e *e2eImpl) SetAKSName(aksName string) {
 }
 
 func (e *e2eImpl) AddStep(step pipeline.Step, rg int) {
-	e.pipeline.ResourceGroups[rg].Steps = append(e.pipeline.ResourceGroups[rg].Steps, &step)
+	e.pipeline.ResourceGroups[rg].Steps = append(e.pipeline.ResourceGroups[rg].Steps, step)
 }
 
 func (e *e2eImpl) SetConfig(updates config.Variables) {

diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go
@@ -35,17 +35,13 @@ func TestE2EMake(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "test",
-		Action:  "Shell",
-		Command: "make test",
-		Variables: []pipeline.Variable{
-			{
-				Name:      "TEST_ENV",
-				ConfigRef: "test_env",
-			},
-		},
-	}, 0)
+	e2eImpl.AddStep(
+		pipeline.NewShellStep("test", "make test").WithVariables(pipeline.Variable{
+			Name:      "TEST_ENV",
+			ConfigRef: "test_env",
+		}),
+		0,
+	)
 
 	e2eImpl.SetConfig(config.Variables{"defaults": config.Variables{"test_env": "test_env"}})
 
@@ -68,11 +64,7 @@ func TestE2EKubernetes(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "test",
-		Action:  "Shell",
-		Command: "kubectl get namespaces",
-	}, 0)
+	e2eImpl.AddStep(pipeline.NewShellStep("test", "kubectl get namespaces"), 0)
 	e2eImpl.SetAKSName("aro-hcp-aks")
 
 	e2eImpl.SetConfig(config.Variables{"defaults": config.Variables{"rg": "hcp-underlay-dev-svc"}})
@@ -88,13 +80,7 @@ func TestE2EArmDeploy(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "test",
-		Action:     "ARM",
-		Template:   "test.bicep",
-		Parameters: "test.bicepparm",
-	}, 0)
-
+	e2eImpl.AddStep(pipeline.NewARMStep("test", "test.bicep", "test.bicepparm"), 0)
 	cleanup := e2eImpl.UseRandomRG()
 	defer func() {
 		err := cleanup()
@@ -140,11 +126,10 @@ func TestE2EShell(t *testing.T) {
 
 	e2eImpl := newE2E(tmpDir)
 
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "/bin/echo ${PWD} > env.txt",
-	}, 0)
+	e2eImpl.AddStep(
+		pipeline.NewShellStep("readInput", "/bin/echo ${PWD} > env.txt"),
+		0,
+	)
 
 	persistAndRun(t, &e2eImpl)
 
@@ -161,27 +146,20 @@ func TestE2EArmDeployWithOutput(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "createZone",
-		Action:     "ARM",
-		Template:   "test.bicep",
-		Parameters: "test.bicepparm",
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${zoneName} > env.txt",
-		Variables: []pipeline.Variable{
-			{
+
+	e2eImpl.AddStep(pipeline.NewARMStep("createZone", "test.bicep", "test.bicepparm"), 0)
+
+	e2eImpl.AddStep(pipeline.NewShellStep(
+		"readInput", "echo ${zoneName} > env.txt",
+	).WithVariables(
+		pipeline.Variable{
+			Name: "zoneName",
+			Input: &pipeline.Input{
 				Name: "zoneName",
-				Input: &pipeline.Input{
-					Name: "zoneName",
-					Step: "createZone",
-				},
+				Step: "createZone",
 			},
 		},
-	}, 0)
+	), 0)
 
 	cleanup := e2eImpl.UseRandomRG()
 	defer func() {
@@ -212,43 +190,26 @@ func TestE2EArmDeployWithOutputToArm(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterA",
-		Action:     "ARM",
-		Template:   "testa.bicep",
-		Parameters: "testa.bicepparm",
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterB",
-		Action:     "ARM",
-		Template:   "testb.bicep",
-		Parameters: "testb.bicepparm",
-		Variables: []pipeline.Variable{
-			{
-				Name: "parameterB",
-				Input: &pipeline.Input{
-					Name: "parameterA",
-					Step: "parameterA",
-				},
-			},
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterA", "testa.bicep", "testa.bicepparm"), 0)
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterB", "testb.bicep", "testb.bicepparm").WithVariables(pipeline.Variable{
+		Name: "parameterB",
+		Input: &pipeline.Input{
+			Name: "parameterA",
+			Step: "parameterA",
 		},
-	}, 0)
-
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${end} > env.txt",
-		Variables: []pipeline.Variable{
-			{
-				Name: "end",
-				Input: &pipeline.Input{
-					Name: "parameterC",
-					Step: "parameterB",
-				},
+	}), 0)
+
+	e2eImpl.AddStep(pipeline.NewShellStep(
+		"readInput", "echo ${end} > env.txt",
+	).WithVariables(
+		pipeline.Variable{
+			Name: "end",
+			Input: &pipeline.Input{
+				Name: "parameterC",
+				Step: "parameterB",
 			},
 		},
-	}, 0)
+	), 0)
 
 	e2eImpl.AddBicepTemplate(`
 param parameterA string
@@ -290,29 +251,19 @@ func TestE2EArmDeployWithOutputRGOverlap(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	e2eImpl.AddStep(pipeline.Step{
-		Name:       "parameterA",
-		Action:     "ARM",
-		Template:   "testa.bicep",
-		Parameters: "testa.bicepparm",
-	}, 0)
+	e2eImpl.AddStep(pipeline.NewARMStep("parameterA", "testa.bicep", "testa.bicepparm"), 0)
 
 	e2eImpl.AddResourceGroup()
 
-	e2eImpl.AddStep(pipeline.Step{
-		Name:    "readInput",
-		Action:  "Shell",
-		Command: "echo ${end} > env.txt",
-		Variables: []pipeline.Variable{
-			{
-				Name: "end",
-				Input: &pipeline.Input{
-					Name: "parameterA",
-					Step: "parameterA",
-				},
+	e2eImpl.AddStep(pipeline.NewShellStep("readInput", "echo ${end} > env.txt").WithVariables(
+		pipeline.Variable{
+			Name: "end",
+			Input: &pipeline.Input{
+				Name: "parameterA",
+				Step: "parameterA",
 			},
 		},
-	}, 1)
+	), 1)
 
 	e2eImpl.AddBicepTemplate(`
 param parameterA string

diff --git a/tooling/templatize/pkg/ev2/pipeline.go b/tooling/templatize/pkg/ev2/pipeline.go
@@ -68,16 +68,17 @@ func readReferencedPipelineFiles(p *pipeline.Pipeline) (map[string][]byte, error
 	referencedFiles := make(map[string][]byte)
 	for _, rg := range p.ResourceGroups {
 		for _, step := range rg.Steps {
-			if step.Parameters != "" {
-				absFilePath, err := p.AbsoluteFilePath(step.Parameters)
+			switch concreteStep := step.(type) {
+			case *pipeline.ARMStep:
+				absFilePath, err := p.AbsoluteFilePath(concreteStep.Parameters)
 				if err != nil {
-					return nil, fmt.Errorf("failed to get absolute file path for %q: %w", step.Parameters, err)
+					return nil, fmt.Errorf("failed to get absolute file path for %q: %w", concreteStep.Parameters, err)
 				}
 				paramFileContent, err := os.ReadFile(absFilePath)
 				if err != nil {
-					return nil, fmt.Errorf("failed to read parameter file %q: %w", step.Parameters, err)
+					return nil, fmt.Errorf("failed to read parameter file %q: %w", concreteStep.Parameters, err)
 				}
-				referencedFiles[step.Parameters] = paramFileContent
+				referencedFiles[concreteStep.Parameters] = paramFileContent
 			}
 		}
 	}
@@ -94,18 +95,19 @@ func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]by
 	for _, rg := range processingPipeline.ResourceGroups {
 		for _, step := range rg.Steps {
 			// preprocess the parameters file with scopebinding variables
-			if step.Parameters != "" {
-				paramFileContent, ok := referencedFiles[step.Parameters]
+			switch concreteStep := step.(type) {
+			case *pipeline.ARMStep:
+				paramFileContent, ok := referencedFiles[concreteStep.Parameters]
 				if !ok {
-					return nil, nil, fmt.Errorf("parameter file %q not found", step.Parameters)
+					return nil, nil, fmt.Errorf("parameter file %q not found", concreteStep.Parameters)
 				}
 				preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundBicepParamVars)
 				if err != nil {
 					return nil, nil, err
 				}
-				newParameterFilePath := buildPrefixedFilePath(step.Parameters, precompiledPrefix)
+				newParameterFilePath := buildPrefixedFilePath(concreteStep.Parameters, precompiledPrefix)
 				processedFiles[newParameterFilePath] = preprocessedBytes
-				step.Parameters = newParameterFilePath
+				concreteStep.Parameters = newParameterFilePath
 			}
 		}
 	}

diff --git a/tooling/templatize/pkg/pipeline/arm.go b/tooling/templatize/pkg/pipeline/arm.go
@@ -31,7 +31,7 @@ func newArmClient(subscriptionID, region string) *armClient {
 	}
 }
 
-func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, rgName string, step *Step, input map[string]output) (output, error) {
+func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, rgName string, step *ARMStep, input map[string]output) (output, error) {
 	// Ensure resourcegroup exists
 	err := a.ensureResourceGroupExists(ctx, rgName, options.NoPersist)
 	if err != nil {
@@ -69,7 +69,7 @@ func printChanges(t armresources.ChangeType, changes []*armresources.WhatIfChang
 		}
 	}
 }
-func doDryRun(ctx context.Context, client *armresources.DeploymentsClient, rgName string, step *Step, vars config.Variables, input map[string]output) (output, error) {
+func doDryRun(ctx context.Context, client *armresources.DeploymentsClient, rgName string, step *ARMStep, vars config.Variables, input map[string]output) (output, error) {
 
 	logger := logr.FromContextOrDiscard(ctx)
 
@@ -126,7 +126,7 @@ func doDryRun(ctx context.Context, client *armresources.DeploymentsClient, rgNam
 	return nil, nil
 }
 
-func doWaitForDeployment(ctx context.Context, client *armresources.DeploymentsClient, rgName string, step *Step, vars config.Variables, input map[string]output) (output, error) {
+func doWaitForDeployment(ctx context.Context, client *armresources.DeploymentsClient, rgName string, step *ARMStep, vars config.Variables, input map[string]output) (output, error) {
 	logger := logr.FromContextOrDiscard(ctx)
 
 	inputValues, err := getInputValues(step.Variables, input)

diff --git a/tooling/templatize/pkg/pipeline/common_test.go b/tooling/templatize/pkg/pipeline/common_test.go
@@ -4,9 +4,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"gotest.tools/v3/assert"
+
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"gotest.tools/v3/assert"
 
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
@@ -31,7 +32,7 @@ func TestDeepCopy(t *testing.T) {
 	assert.Assert(t, pipeline != pipelineCopy, "expected pipeline and copy to be different")
 	assert.Equal(t, pipelineCopy.PipelineFilePath(), newPipelinePath, "expected pipeline copy to have new path")
 
-	if diff := cmp.Diff(pipeline, pipelineCopy, cmpopts.IgnoreUnexported(Pipeline{}, Step{})); diff != "" {
+	if diff := cmp.Diff(pipeline, pipelineCopy, cmpopts.IgnoreUnexported(Pipeline{}, ShellStep{}, ARMStep{})); diff != "" {
 		t.Errorf("got diffs after pipeline deep copy: %v", diff)
 	}
 }

diff --git a/tooling/templatize/pkg/pipeline/inspect.go b/tooling/templatize/pkg/pipeline/inspect.go
@@ -8,7 +8,7 @@ import (
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
 
-type StepInspectScope func(*Step, *InspectOptions, io.Writer) error
+type StepInspectScope func(Step, *InspectOptions, io.Writer) error
 
 func NewStepInspectScopes() map[string]StepInspectScope {
 	return map[string]StepInspectScope{
@@ -41,7 +41,7 @@ func NewInspectOptions(vars config.Variables, region, step, scope, format string
 func (p *Pipeline) Inspect(ctx context.Context, options *InspectOptions, writer io.Writer) error {
 	for _, rg := range p.ResourceGroups {
 		for _, step := range rg.Steps {
-			if step.Name == options.Step {
+			if step.StepName() == options.Step {
 				if inspectFunc, ok := options.ScopeFunctions[options.Scope]; ok {
 					err := inspectFunc(step, options, writer)
 					if err != nil {
@@ -57,14 +57,14 @@ func (p *Pipeline) Inspect(ctx context.Context, options *InspectOptions, writer
 	return fmt.Errorf("step %q not found", options.Step)
 }
 
-func inspectVars(s *Step, options *InspectOptions, writer io.Writer) error {
+func inspectVars(s Step, options *InspectOptions, writer io.Writer) error {
 	var envVars map[string]string
 	var err error
-	switch s.Action {
-	case "Shell":
-		envVars, err = s.mapStepVariables(options.Vars)
+	switch step := s.(type) {
+	case *ShellStep:
+		envVars, err = step.mapStepVariables(options.Vars)
 	default:
-		return fmt.Errorf("inspecting step variables not implemented for action type %s", s.Action)
+		return fmt.Errorf("inspecting step variables not implemented for action type %s", s.ActionType())
 	}
 	if err != nil {
 		return err