diff --git a/Makefile b/Makefile index c9b405af9..46d0a6ad4 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ all: test lint # There is currently no convenient way to run tests against a whole Go workspace # https://github.com/golang/go/issues/50745 test: - go list -f '{{.Dir}}/...' -m |RUN_TEMPLATIZE_E2E=true xargs go test -timeout 120s -tags=$(GOTAGS) -cover + go list -f '{{.Dir}}/...' -m |RUN_TEMPLATIZE_E2E=true xargs go test -timeout 1200s -tags=$(GOTAGS) -cover .PHONY: test # There is currently no convenient way to run golangci-lint against a whole Go workspace diff --git a/backend/pipeline.yaml b/backend/pipeline.yaml index 33f357912..2fb5a91d2 100644 --- a/backend/pipeline.yaml +++ b/backend/pipeline.yaml @@ -8,7 +8,7 @@ resourceGroups: - name: deploy action: Shell command: make deploy - env: + variables: - name: ARO_HCP_IMAGE_ACR configRef: svcAcrName - name: LOCATION diff --git a/frontend/pipeline.yaml b/frontend/pipeline.yaml index 88794bcf5..f342f5302 100644 --- a/frontend/pipeline.yaml +++ b/frontend/pipeline.yaml @@ -9,10 +9,10 @@ resourceGroups: action: Shell command: make deploy dryRun: - envVars: + variables: - name: HELM_DRY_RUN value: "--dry-run=server --debug" - env: + variables: - name: ARO_HCP_IMAGE_ACR configRef: svcAcrName - name: LOCATION diff --git a/maestro/server/pipeline.yaml b/maestro/server/pipeline.yaml index 394a818ce..982799129 100644 --- a/maestro/server/pipeline.yaml +++ b/maestro/server/pipeline.yaml @@ -8,7 +8,7 @@ resourceGroups: - name: deploy action: Shell command: make deploy - env: + variables: - name: EVENTGRID_NAME configRef: maestro.eventGrid.name - name: REGION_RG diff --git a/tooling/templatize/internal/end2end/e2e.go b/tooling/templatize/internal/end2end/e2e.go index a7149116c..6edb6d387 100644 --- a/tooling/templatize/internal/end2end/e2e.go +++ b/tooling/templatize/internal/end2end/e2e.go @@ -1,12 +1,17 @@ package testutil import ( + "context" "fmt" - "math/rand/v2" "os" + "math/rand/v2" + "gopkg.in/yaml.v2" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" ) @@ -19,21 +24,28 @@ func shouldRunE2E() bool { type E2E interface { SetConfig(updates config.Variables) - UseRandomRG() + UseRandomRG() func() error + AddBicepTemplate(template, templateFileName, paramfile, paramfileName string) AddStep(step pipeline.Step) SetOSArgs() Persist() error } +type bicepTemplate struct { + bicepFile string + bicepFileName string + paramFile string + paramFileName string +} + type e2eImpl struct { - config config.Variables - makefile string - pipeline pipeline.Pipeline - bicepFile string - paramFile string - schema string - tmpdir string - rgName string + config config.Variables + makefile string + pipeline pipeline.Pipeline + biceps []bicepTemplate + schema string + tmpdir string + rgName string } var _ E2E = &e2eImpl{} @@ -71,13 +83,14 @@ func newE2E(tmpdir string) e2eImpl { }, }, rgName: defaultRgName, + biceps: []bicepTemplate{}, } imp.SetOSArgs() return imp } -func (e *e2eImpl) UseRandomRG() { +func (e *e2eImpl) UseRandomRG() func() error { chars := []rune("abcdefghijklmnopqrstuvwxyz0123456789") rg := "templatize-e2e-" @@ -87,6 +100,23 @@ func (e *e2eImpl) UseRandomRG() { } e.rgName = rg e.SetConfig(config.Variables{"defaults": config.Variables{"rg": rg}}) + + return func() error { + subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)") + if err != nil { + return err + } + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return err + } + rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil) + if err != nil { + return err + } + _, err = rgClient.BeginDelete(context.Background(), e.rgName, nil) + return err + } } func (e *e2eImpl) SetOSArgs() { @@ -111,16 +141,28 @@ func (e *e2eImpl) SetConfig(updates config.Variables) { config.MergeVariables(e.config, updates) } -func (e *e2eImpl) Persist() error { - if e.bicepFile != "" && e.paramFile != "" { - err := os.WriteFile(e.tmpdir+"/test.bicep", []byte(e.bicepFile), 0644) - if err != nil { - return err - } +func (e *e2eImpl) AddBicepTemplate(template, templateFileName, paramfile, paramfileName string) { + e.biceps = append(e.biceps, bicepTemplate{ + bicepFile: template, + bicepFileName: templateFileName, + paramFile: paramfile, + paramFileName: paramfileName, + }) +} - err = os.WriteFile(e.tmpdir+"/test.bicepparm", []byte(e.paramFile), 0644) - if err != nil { - return err +func (e *e2eImpl) Persist() error { + if len(e.biceps) != 0 { + for _, b := range e.biceps { + + err := os.WriteFile(e.tmpdir+"/"+b.bicepFileName, []byte(b.bicepFile), 0644) + if err != nil { + return err + } + + err = os.WriteFile(e.tmpdir+"/"+b.paramFileName, []byte(b.paramFile), 0644) + if err != nil { + return err + } } } diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go index 8e6c4e877..d74625069 100644 --- a/tooling/templatize/internal/end2end/e2e_test.go +++ b/tooling/templatize/internal/end2end/e2e_test.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) func persistAndRun(t *testing.T, e2eImpl E2E) { @@ -39,7 +38,7 @@ func TestE2EMake(t *testing.T) { Name: "test", Action: "Shell", Command: "make test", - Env: []pipeline.EnvVar{ + Variables: []pipeline.Variable{ { Name: "TEST_ENV", ConfigRef: "test_env", @@ -95,18 +94,23 @@ func TestE2EArmDeploy(t *testing.T) { Parameters: "test.bicepparm", }) - e2eImpl.UseRandomRG() + cleanup := e2eImpl.UseRandomRG() + defer func() { + err := cleanup() + assert.NilError(t, err) + }() - e2eImpl.bicepFile = ` + bicepFile := ` param zoneName string resource symbolicname 'Microsoft.Network/dnsZones@2018-05-01' = { location: 'global' name: zoneName }` - e2eImpl.paramFile = ` + paramFile := ` using 'test.bicep' param zoneName = 'e2etestarmdeploy.foo.bar.example.com' ` + e2eImpl.AddBicepTemplate(bicepFile, "test.bicep", paramFile, "test.bicepparm") persistAndRun(t, &e2eImpl) @@ -117,29 +121,161 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com' cred, err := azidentity.NewDefaultAzureCredential(nil) assert.NilError(t, err) - rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil) - assert.NilError(t, err) - - existence, err := rgClient.CheckExistence(context.Background(), e2eImpl.rgName, nil) - assert.NilError(t, err) - assert.Assert(t, existence.Success) - zonesClient, err := armdns.NewZonesClient(subsriptionID, cred, nil) assert.NilError(t, err) zoneResp, err := zonesClient.Get(context.Background(), e2eImpl.rgName, "e2etestarmdeploy.foo.bar.example.com", nil) assert.NilError(t, err) assert.Equal(t, *zoneResp.Name, "e2etestarmdeploy.foo.bar.example.com") +} - delResponse, err := zonesClient.BeginDelete(context.Background(), e2eImpl.rgName, "e2etestarmdeploy.foo.bar.example.com", nil) - assert.NilError(t, err) +func TestE2EShell(t *testing.T) { + if !shouldRunE2E() { + t.Skip("Skipping end-to-end tests") + } + + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: "/usr/bin/echo ${PWD} > env.txt", + }) - _, err = delResponse.PollUntilDone(context.Background(), nil) + persistAndRun(t, &e2eImpl) + + io, err := os.ReadFile(tmpDir + "/env.txt") assert.NilError(t, err) + assert.Equal(t, string(io), tmpDir+"\n") +} + +func TestE2EArmDeployWithOutput(t *testing.T) { + if !shouldRunE2E() { + t.Skip("Skipping end-to-end tests") + } + + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + e2eImpl.AddStep(pipeline.Step{ + Name: "createZone", + Action: "ARM", + Template: "test.bicep", + Parameters: "test.bicepparm", + }) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: "echo ${zoneName} > env.txt", + Variables: []pipeline.Variable{ + { + Name: "zoneName", + Input: &pipeline.Input{ + Name: "zoneName", + Step: "createZone", + }, + }, + }, + }) + + cleanup := e2eImpl.UseRandomRG() + defer func() { + err := cleanup() + assert.NilError(t, err) + }() + + bicepFile := ` +param zoneName string +output zoneName string = zoneName` + paramFile := ` +using 'test.bicep' +param zoneName = 'e2etestarmdeploy.foo.bar.example.com' +` + e2eImpl.AddBicepTemplate(bicepFile, "test.bicep", paramFile, "test.bicepparm") + persistAndRun(t, &e2eImpl) - rgDelResponse, err := rgClient.BeginDelete(context.Background(), e2eImpl.rgName, nil) + io, err := os.ReadFile(tmpDir + "/env.txt") assert.NilError(t, err) + assert.Equal(t, string(io), "e2etestarmdeploy.foo.bar.example.com\n") +} + +func TestE2EArmDeployWithOutputToArm(t *testing.T) { + if !shouldRunE2E() { + t.Skip("Skipping end-to-end tests") + } - _, err = rgDelResponse.PollUntilDone(context.Background(), nil) + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + e2eImpl.AddStep(pipeline.Step{ + Name: "parameterA", + Action: "ARM", + Template: "testa.bicep", + Parameters: "testa.bicepparm", + }) + + e2eImpl.AddStep(pipeline.Step{ + Name: "parameterB", + Action: "ARM", + Template: "testb.bicep", + Parameters: "testb.bicepparm", + Variables: []pipeline.Variable{ + { + Name: "parameterB", + Input: &pipeline.Input{ + Name: "parameterA", + Step: "parameterA", + }, + }, + }, + }) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: "echo ${end} > env.txt", + Variables: []pipeline.Variable{ + { + Name: "end", + Input: &pipeline.Input{ + Name: "parameterC", + Step: "parameterB", + }, + }, + }, + }) + + e2eImpl.AddBicepTemplate(` +param parameterA string +output parameterA string = parameterA`, + "testa.bicep", + ` +using 'testa.bicep' +param parameterA = 'Hello Bicep'`, + "testa.bicepparm") + + e2eImpl.AddBicepTemplate(` +param parameterB string +output parameterC string = parameterB +`, + "testb.bicep", + ` +using 'testb.bicep' +param parameterB = '{{ .parameterB }}' +`, + "testb.bicepparm") + + cleanup := e2eImpl.UseRandomRG() + defer func() { + err := cleanup() + assert.NilError(t, err) + }() + persistAndRun(t, &e2eImpl) + + io, err := os.ReadFile(tmpDir + "/env.txt") assert.NilError(t, err) + assert.Equal(t, string(io), "Hello Bicep\n") } diff --git a/tooling/templatize/pkg/pipeline/arm.go b/tooling/templatize/pkg/pipeline/arm.go index 888bce862..5e72e312a 100644 --- a/tooling/templatize/pkg/pipeline/arm.go +++ b/tooling/templatize/pkg/pipeline/arm.go @@ -10,56 +10,82 @@ import ( "github.com/go-logr/logr" ) -func (s *Step) runArmStep(ctx context.Context, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +type armClient struct { + creds *azidentity.DefaultAzureCredential + SubscriptionID string + Region string +} + +func newArmClient(subscriptionID, region string) *armClient { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil + } + return &armClient{ + creds: cred, + SubscriptionID: subscriptionID, + Region: region, + } +} + +func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, rgName string, step *Step, input map[string]output) (output, error) { logger := logr.FromContextOrDiscard(ctx) + inputValues, err := getInputValues(step.Variables, input) + if err != nil { + return nil, fmt.Errorf("failed to get input values: %w", err) + } // Transform Bicep to ARM - deploymentProperties, err := transformBicepToARM(ctx, s.Parameters, options.Vars) + deploymentProperties, err := transformBicepToARM(ctx, step.Parameters, options.Vars, inputValues) if err != nil { - return fmt.Errorf("failed to transform Bicep to ARM: %w", err) + return nil, fmt.Errorf("failed to transform Bicep to ARM: %w", err) } // Create the deployment - deploymentName := s.Name deployment := armresources.Deployment{ Properties: deploymentProperties, } // Ensure resourcegroup exists - err = s.ensureResourceGroupExists(ctx, executionTarget) + err = a.ensureResourceGroupExists(ctx, rgName) if err != nil { - return fmt.Errorf("failed to ensure resource group exists: %w", err) + return nil, fmt.Errorf("failed to ensure resource group exists: %w", err) } // TODO handle dry-run // Run deployment - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return fmt.Errorf("failed to obtain a credential: %w", err) - } - - client, err := armresources.NewDeploymentsClient(executionTarget.GetSubscriptionID(), cred, nil) + client, err := armresources.NewDeploymentsClient(a.SubscriptionID, a.creds, nil) if err != nil { - return fmt.Errorf("failed to create deployments client: %w", err) + return nil, fmt.Errorf("failed to create deployments client: %w", err) } - poller, err := client.BeginCreateOrUpdate(ctx, executionTarget.GetResourceGroup(), deploymentName, deployment, nil) + poller, err := client.BeginCreateOrUpdate(ctx, rgName, step.Name, deployment, nil) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + return nil, fmt.Errorf("failed to create deployment: %w", err) } - logger.Info("Deployment started", "deployment", deploymentName) + logger.Info("Deployment started", "deployment", step.Name) // Wait for completion resp, err := poller.PollUntilDone(ctx, nil) if err != nil { - return fmt.Errorf("failed to wait for deployment completion: %w", err) + return nil, fmt.Errorf("failed to wait for deployment completion: %w", err) } - logger.Info("Deployment finished successfully", "deployment", deploymentName, "responseId", *resp.ID) - return nil + logger.Info("Deployment finished successfully", "deployment", step.Name, "responseId", *resp.ID) + + if resp.Properties.Outputs != nil { + if outputMap, ok := resp.Properties.Outputs.(map[string]any); ok { + returnMap := armOutput{} + for k, v := range outputMap { + returnMap[k] = v + } + return returnMap, nil + } + } + return nil, nil } -func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget ExecutionTarget) error { +func (a *armClient) ensureResourceGroupExists(ctx context.Context, rgName string) error { // Create a new Azure identity client cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -67,7 +93,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex } // Create a new ARM client - client, err := armresources.NewResourceGroupsClient(executionTarget.GetSubscriptionID(), cred, nil) + client, err := armresources.NewResourceGroupsClient(a.SubscriptionID, cred, nil) if err != nil { return fmt.Errorf("failed to create ARM client: %w", err) } @@ -77,14 +103,14 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex tags := map[string]*string{ "persist": to.Ptr("true"), } - _, err = client.Get(ctx, executionTarget.GetResourceGroup(), nil) + _, err = client.Get(ctx, rgName, nil) if err != nil { // Create the resource group resourceGroup := armresources.ResourceGroup{ - Location: to.Ptr(executionTarget.GetRegion()), + Location: to.Ptr(a.Region), Tags: tags, } - _, err = client.CreateOrUpdate(ctx, executionTarget.GetResourceGroup(), resourceGroup, nil) + _, err = client.CreateOrUpdate(ctx, rgName, resourceGroup, nil) if err != nil { return fmt.Errorf("failed to create resource group: %w", err) } @@ -92,7 +118,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex patchResourceGroup := armresources.ResourceGroupPatchable{ Tags: tags, } - _, err = client.Update(ctx, executionTarget.GetResourceGroup(), patchResourceGroup, nil) + _, err = client.Update(ctx, rgName, patchResourceGroup, nil) if err != nil { return fmt.Errorf("failed to update resource group: %w", err) } diff --git a/tooling/templatize/pkg/pipeline/bicep.go b/tooling/templatize/pkg/pipeline/bicep.go index 7b8ca60d5..1b4848890 100644 --- a/tooling/templatize/pkg/pipeline/bicep.go +++ b/tooling/templatize/pkg/pipeline/bicep.go @@ -14,11 +14,20 @@ import ( "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" ) -func transformBicepToARM(ctx context.Context, bicepParameterTemplateFile string, vars config.Variables) (*armresources.DeploymentProperties, error) { +func transformBicepToARM(ctx context.Context, bicepParameterTemplateFile string, vars config.Variables, inputs map[string]any) (*armresources.DeploymentProperties, error) { // preprocess bicep parameter file and store it - bicepParamContent, error := config.PreprocessFile(bicepParameterTemplateFile, vars) - if error != nil { - return nil, fmt.Errorf("failed to preprocess file: %w", error) + + combinedVars := make(map[string]any) + for k, v := range vars { + combinedVars[k] = v + } + for k, v := range inputs { + combinedVars[k] = v + } + + bicepParamContent, err := config.PreprocessFile(bicepParameterTemplateFile, combinedVars) + if err != nil { + return nil, fmt.Errorf("failed to preprocess file: %w", err) } bicepParamBaseDir := filepath.Dir(bicepParameterTemplateFile) bicepParamFile, err := os.CreateTemp(bicepParamBaseDir, "bicep-params-*.bicepparam") diff --git a/tooling/templatize/pkg/pipeline/inspect_test.go b/tooling/templatize/pkg/pipeline/inspect_test.go index dc308715e..744ec4707 100644 --- a/tooling/templatize/pkg/pipeline/inspect_test.go +++ b/tooling/templatize/pkg/pipeline/inspect_test.go @@ -23,7 +23,7 @@ func TestInspectVars(t *testing.T) { name: "basic", caseStep: &Step{ Action: "Shell", - Env: []EnvVar{ + Variables: []Variable{ { Name: "FOO", ConfigRef: "foo", @@ -42,7 +42,7 @@ func TestInspectVars(t *testing.T) { name: "makefile", caseStep: &Step{ Action: "Shell", - Env: []EnvVar{ + Variables: []Variable{ { Name: "FOO", ConfigRef: "foo", diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index 492522665..c376427d4 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -46,6 +46,30 @@ type PipelineRunOptions struct { SubsciptionLookupFunc subsciptionLookup } +type armOutput map[string]any + +type output interface { + GetValue(key string) (*outPutValue, error) +} + +type outPutValue struct { + Type string `yaml:"type"` + Value any `yaml:"value"` +} + +func (o armOutput) GetValue(key string) (*outPutValue, error) { + if v, ok := o[key]; ok { + if innerValue, innerConversionOk := v.(map[string]any); innerConversionOk { + returnValue := outPutValue{ + Type: innerValue["type"].(string), + Value: innerValue["value"], + } + return &returnValue, nil + } + } + return nil, fmt.Errorf("key %q not found", key) +} + func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { logger := logr.FromContextOrDiscard(ctx) @@ -94,6 +118,8 @@ func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, executionTarget ExecutionTarget) error { logger := logr.FromContextOrDiscard(ctx) + outPuts := make(map[string]output) + kubeconfigFile, err := executionTarget.KubeConfig(ctx) if kubeconfigFile != "" { defer func() { @@ -107,7 +133,7 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e for _, step := range rg.Steps { // execute - err := step.run( + output, err := step.run( logr.NewContext( ctx, logger.WithValues( @@ -119,18 +145,22 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e ), kubeconfigFile, executionTarget, options, + outPuts, ) if err != nil { return err } + if output != nil { + outPuts[step.Name] = output + } } return nil } -func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions, outPuts map[string]output) (output, error) { if options.Step != "" && s.Name != options.Step { // skip steps that don't match the specified step name - return nil + return nil, nil } fmt.Println("\n---------------------") if options.DryRun { @@ -141,11 +171,19 @@ func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget E switch s.Action { case "Shell": - return s.runShellStep(ctx, kubeconfigFile, options) + return nil, s.runShellStep(ctx, kubeconfigFile, options, outPuts) case "ARM": - return s.runArmStep(ctx, executionTarget, options) + a := newArmClient(executionTarget.GetSubscriptionID(), executionTarget.GetRegion()) + if a == nil { + return nil, fmt.Errorf("failed to create ARM client") + } + output, err := a.runArmStep(ctx, options, executionTarget.GetResourceGroup(), s, outPuts) + if err != nil { + return nil, fmt.Errorf("failed to run ARM step: %w", err) + } + return output, nil default: - return fmt.Errorf("unsupported action type %q", s.Action) + return nil, fmt.Errorf("unsupported action type %q", s.Action) } } @@ -203,3 +241,21 @@ func (rg *ResourceGroup) Validate() error { } return nil } + +func getInputValues(configuredVariables []Variable, inputs map[string]output) (map[string]any, error) { + values := make(map[string]any) + for _, i := range configuredVariables { + if i.Input != nil { + if v, found := inputs[i.Input.Step]; found { + value, err := v.GetValue(i.Input.Name) + if err != nil { + return nil, fmt.Errorf("failed to get value for input %s.%s: %w", i.Input.Step, i.Input.Name, err) + } + values[i.Name] = value.Value + } else { + return nil, fmt.Errorf("step %s not found in provided outputs", i.Input.Step) + } + } + } + return values, nil +} diff --git a/tooling/templatize/pkg/pipeline/run_test.go b/tooling/templatize/pkg/pipeline/run_test.go index 41992e6b9..650cd402f 100644 --- a/tooling/templatize/pkg/pipeline/run_test.go +++ b/tooling/templatize/pkg/pipeline/run_test.go @@ -17,7 +17,7 @@ func TestStepRun(t *testing.T) { fooundOutput = output }, } - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}, nil) assert.NilError(t, err) assert.Equal(t, fooundOutput, "hello\n") } @@ -27,11 +27,11 @@ func TestStepRunSkip(t *testing.T) { Name: "step", } // this should skip - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}, nil) assert.NilError(t, err) // this should fail - err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}) + _, err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}, nil) assert.Error(t, err, "unsupported action type \"\"") } @@ -280,3 +280,47 @@ func TestPipelineRun(t *testing.T) { assert.NilError(t, err) assert.Equal(t, foundOutput, "hello\n") } + +func TestArmGetValue(t *testing.T) { + output := armOutput{ + "zoneName": map[string]any{ + "type": "String", + "value": "test", + }, + } + + value, err := output.GetValue("zoneName") + assert.Equal(t, value.Value, "test") + assert.Equal(t, value.Type, "String") + assert.NilError(t, err) +} + +func TestAddInputVars(t *testing.T) { + mapOutput := map[string]output{} + mapOutput["step1"] = armOutput{ + "output1": map[string]any{ + "type": "String", + "value": "value1", + }, + } + s := &Step{ + Variables: []Variable{{ + Name: "input1", + Input: &Input{ + Name: "output1", + Step: "step1", + }, + }, + }} + + envVars, err := getInputValues(s.Variables, mapOutput) + assert.NilError(t, err) + assert.DeepEqual(t, envVars, map[string]any{"input1": "value1"}) + + _, err = getInputValues([]Variable{ + { + Input: &Input{Step: "foobar"}, + }, + }, mapOutput) + assert.Error(t, err, "step foobar not found in provided outputs") +} diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go index 48730b94e..0427c8dc0 100644 --- a/tooling/templatize/pkg/pipeline/shell.go +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -15,13 +15,13 @@ import ( func (s *Step) createCommand(ctx context.Context, dryRun bool, envVars map[string]string) (*exec.Cmd, bool) { var scriptCommand string = s.Command if dryRun { - if s.DryRun.Command == "" && s.DryRun.EnvVars == nil { + if s.DryRun.Command == "" && s.DryRun.Variables == nil { return nil, true } if s.DryRun.Command != "" { scriptCommand = s.DryRun.Command } - for _, e := range s.DryRun.EnvVars { + for _, e := range s.DryRun.Variables { envVars[e.Name] = e.Value } } @@ -34,7 +34,7 @@ func buildBashScript(command string) string { return fmt.Sprintf("set -o errexit -o nounset -o pipefail\n%s", command) } -func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { +func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions, inputs map[string]output) error { if s.outputFunc == nil { s.outputFunc = func(output string) { fmt.Println(output) @@ -52,6 +52,14 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options envVars := utils.GetOsVariable() maps.Copy(envVars, stepVars) + + inputValues, err := getInputValues(s.Variables, inputs) + if err != nil { + return fmt.Errorf("failed to get input values: %w", err) + } + for k, v := range inputValues { + envVars[k] = utils.AnyToString(v) + } // execute the command cmd, skipCommand := s.createCommand(ctx, options.DryRun, envVars) if skipCommand { @@ -76,12 +84,14 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options func (s *Step) mapStepVariables(vars config.Variables) (map[string]string, error) { envVars := make(map[string]string) - for _, e := range s.Env { - value, found := vars.GetByPath(e.ConfigRef) - if !found { - return nil, fmt.Errorf("failed to lookup config reference %s for %s", e.ConfigRef, e.Name) + for _, e := range s.Variables { + if e.ConfigRef != "" { // not all Variables are Environment variables + value, found := vars.GetByPath(e.ConfigRef) + if !found { + return nil, fmt.Errorf("failed to lookup config reference %s for %s", e.ConfigRef, e.Name) + } + envVars[e.Name] = utils.AnyToString(value) } - envVars[e.Name] = utils.AnyToString(value) } return envVars, nil } diff --git a/tooling/templatize/pkg/pipeline/shell_test.go b/tooling/templatize/pkg/pipeline/shell_test.go index 5fe18ef87..a85e6a757 100644 --- a/tooling/templatize/pkg/pipeline/shell_test.go +++ b/tooling/templatize/pkg/pipeline/shell_test.go @@ -45,7 +45,7 @@ func TestCreateCommand(t *testing.T) { step: &Step{ Command: "/usr/bin/echo", DryRun: DryRun{ - EnvVars: []EnvVar{ + Variables: []Variable{ { Name: "DRY_RUN", Value: "true", @@ -96,7 +96,7 @@ func TestMapStepVariables(t *testing.T) { "FOO": "bar", }, step: Step{ - Env: []EnvVar{ + Variables: []Variable{ { Name: "BAZ", ConfigRef: "FOO", @@ -111,7 +111,7 @@ func TestMapStepVariables(t *testing.T) { name: "missing", vars: config.Variables{}, step: Step{ - Env: []EnvVar{ + Variables: []Variable{ { ConfigRef: "FOO", }, @@ -125,7 +125,7 @@ func TestMapStepVariables(t *testing.T) { "FOO": 42, }, step: Step{ - Env: []EnvVar{ + Variables: []Variable{ { Name: "BAZ", ConfigRef: "FOO", @@ -192,7 +192,7 @@ func TestRunShellStep(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := tc.step.runShellStep(context.Background(), "", &PipelineRunOptions{}) + err := tc.step.runShellStep(context.Background(), "", &PipelineRunOptions{}, map[string]output{}) if tc.err != "" { assert.ErrorContains(t, err, tc.err) } else { diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go index 2d1888f97..a24dcfae4 100644 --- a/tooling/templatize/pkg/pipeline/types.go +++ b/tooling/templatize/pkg/pipeline/types.go @@ -1,6 +1,8 @@ package pipeline -import "context" +import ( + "context" +) type subsciptionLookup func(context.Context, string) (string, error) @@ -21,32 +23,31 @@ type ResourceGroup struct { type outPutHandler func(string) type Step struct { - Name string `yaml:"name"` - Action string `yaml:"action"` - Command string `yaml:"command,omitempty"` - Env []EnvVar `yaml:"env,omitempty"` - Template string `yaml:"template,omitempty"` - Parameters string `yaml:"parameters,omitempty"` - DependsOn []string `yaml:"dependsOn,omitempty"` - DryRun DryRun `yaml:"dryRun,omitempty"` - Inputs []Input `yaml:"inputs,omitempty"` - DeploymentLevel string `yaml:"deploymentLevel,omitempty"` + Name string `yaml:"name"` + Action string `yaml:"action"` + Command string `yaml:"command,omitempty"` + Variables []Variable `yaml:"variables,omitempty"` + Template string `yaml:"template,omitempty"` + Parameters string `yaml:"parameters,omitempty"` + DependsOn []string `yaml:"dependsOn,omitempty"` + DryRun DryRun `yaml:"dryRun,omitempty"` + DeploymentLevel string `yaml:"deploymentLevel,omitempty"` outputFunc outPutHandler } type DryRun struct { - EnvVars []EnvVar `yaml:"envVars,omitempty"` - Command string `yaml:"command,omitempty"` + Variables []Variable `yaml:"variables,omitempty"` + Command string `yaml:"command,omitempty"` } -type EnvVar struct { +type Variable struct { Name string `yaml:"name"` ConfigRef string `yaml:"configRef,omitempty"` Value string `yaml:"value,omitempty"` + Input *Input `yaml:"input,omitempty"` } type Input struct { - Name string `yaml:"name"` - Step string `yaml:"step"` - Output string `yaml:"output"` + Name string `yaml:"name"` + Step string `yaml:"step"` } diff --git a/tooling/templatize/testdata/pipeline.yaml b/tooling/templatize/testdata/pipeline.yaml index ddcbef932..f87e53052 100644 --- a/tooling/templatize/testdata/pipeline.yaml +++ b/tooling/templatize/testdata/pipeline.yaml @@ -8,14 +8,14 @@ resourceGroups: - name: deploy action: Shell command: make deploy - env: + variables: - name: MAESTRO_IMAGE configRef: maestro_image - name: dry-run action: Shell command: make deploy dryRun: - envVars: + variables: - name: DRY_RUN value: "A very dry one" - name: svc diff --git a/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml index 177cbc678..7448f9363 100644 --- a/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml +++ b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml @@ -8,14 +8,14 @@ resourceGroups: - name: deploy action: Shell command: make deploy - env: + variables: - name: MAESTRO_IMAGE configRef: maestro_image - name: dry-run action: Shell command: make deploy dryRun: - envVars: + variables: - name: DRY_RUN value: A very dry one - name: svc diff --git a/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml b/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml index 721438d52..48bd192ec 100644 --- a/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml +++ b/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml @@ -8,14 +8,14 @@ resourceGroups: - name: deploy action: Shell command: make deploy - env: + variables: - name: MAESTRO_IMAGE configRef: maestro_image - name: dry-run action: Shell command: make deploy dryRun: - envVars: + variables: - name: DRY_RUN value: "A very dry one" - name: svc