diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go index 30482ab7b..743757d42 100644 --- a/tooling/templatize/internal/end2end/e2e_test.go +++ b/tooling/templatize/internal/end2end/e2e_test.go @@ -143,3 +143,86 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com' _, err = rgDelResponse.PollUntilDone(context.Background(), nil) assert.NilError(t, err) } + +func TestE2EShell(t *testing.T) { + if !shouldRunE2E() { + t.Skip("Skipping end-to-end tests") + } + + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: []string{"/bin/sh", "-c", "/usr/bin/echo ${PWD} > env.txt"}, + }) + + persistAndRun(t, &e2eImpl) + + io, err := os.ReadFile(tmpDir + "/env.txt") + assert.NilError(t, err) + assert.Equal(t, string(io), tmpDir+"\n") +} + +func TestE2EArmDeployWithOutput(t *testing.T) { + if !shouldRunE2E() { + t.Skip("Skipping end-to-end tests") + } + + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + e2eImpl.AddStep(pipeline.Step{ + Name: "createZone", + Action: "ARM", + Template: "test.bicep", + Parameters: "test.bicepparm", + }) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: []string{"/bin/sh", "-c", "echo ${zoneName} > env.txt"}, + Inputs: []pipeline.Input{ + { + Name: "zoneName", + Step: "createZone", + Output: "zoneName", + Type: "string", + }, + }, + }) + + e2eImpl.UseRandomRG() + + e2eImpl.bicepFile = ` +param zoneName string +output zoneName string = zoneName` + e2eImpl.paramFile = ` +using 'test.bicep' +param zoneName = 'e2etestarmdeploy.foo.bar.example.com' +` + + persistAndRun(t, &e2eImpl) + + io, err := os.ReadFile(tmpDir + "/env.txt") + assert.NilError(t, err) + assert.Equal(t, string(io), "e2etestarmdeploy.foo.bar.example.com\n") + + subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)") + assert.NilError(t, err) + + cred, err := azidentity.NewDefaultAzureCredential(nil) + assert.NilError(t, err) + + rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil) + assert.NilError(t, err) + + rgDelResponse, err := rgClient.BeginDelete(context.Background(), e2eImpl.rgName, nil) + assert.NilError(t, err) + + _, err = rgDelResponse.PollUntilDone(context.Background(), nil) + assert.NilError(t, err) +} diff --git a/tooling/templatize/pkg/pipeline/arm.go b/tooling/templatize/pkg/pipeline/arm.go index 888bce862..c2b05aab8 100644 --- a/tooling/templatize/pkg/pipeline/arm.go +++ b/tooling/templatize/pkg/pipeline/arm.go @@ -10,56 +10,78 @@ import ( "github.com/go-logr/logr" ) -func (s *Step) runArmStep(ctx context.Context, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +type armClient struct { + creds *azidentity.DefaultAzureCredential + SubscriptionID string + Region string +} + +func newArmClient(subscriptionID, region string) *armClient { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil + } + return &armClient{ + creds: cred, + SubscriptionID: subscriptionID, + Region: region, + } +} + +func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, deploymentName string, rgName string, paramterFile string, input map[string]output) (output, error) { logger := logr.FromContextOrDiscard(ctx) // Transform Bicep to ARM - deploymentProperties, err := transformBicepToARM(ctx, s.Parameters, options.Vars) + deploymentProperties, err := transformBicepToARM(ctx, paramterFile, options.Vars) if err != nil { - return fmt.Errorf("failed to transform Bicep to ARM: %w", err) + return nil, fmt.Errorf("failed to transform Bicep to ARM: %w", err) } // Create the deployment - deploymentName := s.Name deployment := armresources.Deployment{ Properties: deploymentProperties, } // Ensure resourcegroup exists - err = s.ensureResourceGroupExists(ctx, executionTarget) + err = a.ensureResourceGroupExists(ctx, rgName) if err != nil { - return fmt.Errorf("failed to ensure resource group exists: %w", err) + return nil, fmt.Errorf("failed to ensure resource group exists: %w", err) } // TODO handle dry-run // Run deployment - cred, err := azidentity.NewDefaultAzureCredential(nil) + client, err := armresources.NewDeploymentsClient(a.SubscriptionID, a.creds, nil) if err != nil { - return fmt.Errorf("failed to obtain a credential: %w", err) - } - - client, err := armresources.NewDeploymentsClient(executionTarget.GetSubscriptionID(), cred, nil) - if err != nil { - return fmt.Errorf("failed to create deployments client: %w", err) + return nil, fmt.Errorf("failed to create deployments client: %w", err) } - poller, err := client.BeginCreateOrUpdate(ctx, executionTarget.GetResourceGroup(), deploymentName, deployment, nil) + poller, err := client.BeginCreateOrUpdate(ctx, rgName, deploymentName, deployment, nil) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + return nil, fmt.Errorf("failed to create deployment: %w", err) } logger.Info("Deployment started", "deployment", deploymentName) // Wait for completion resp, err := poller.PollUntilDone(ctx, nil) if err != nil { - return fmt.Errorf("failed to wait for deployment completion: %w", err) + return nil, fmt.Errorf("failed to wait for deployment completion: %w", err) } logger.Info("Deployment finished successfully", "deployment", deploymentName, "responseId", *resp.ID) - return nil + + if resp.Properties.Outputs != nil { + if outputMap, ok := resp.Properties.Outputs.(map[string]any); ok { + returnMap := armOutput{} + for k, v := range outputMap { + returnMap[k] = v + } + return returnMap, nil + } + } + return nil, nil } -func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget ExecutionTarget) error { +func (a *armClient) ensureResourceGroupExists(ctx context.Context, rgName string) error { // Create a new Azure identity client cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -67,7 +89,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex } // Create a new ARM client - client, err := armresources.NewResourceGroupsClient(executionTarget.GetSubscriptionID(), cred, nil) + client, err := armresources.NewResourceGroupsClient(a.SubscriptionID, cred, nil) if err != nil { return fmt.Errorf("failed to create ARM client: %w", err) } @@ -77,14 +99,14 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex tags := map[string]*string{ "persist": to.Ptr("true"), } - _, err = client.Get(ctx, executionTarget.GetResourceGroup(), nil) + _, err = client.Get(ctx, rgName, nil) if err != nil { // Create the resource group resourceGroup := armresources.ResourceGroup{ - Location: to.Ptr(executionTarget.GetRegion()), + Location: to.Ptr(a.Region), Tags: tags, } - _, err = client.CreateOrUpdate(ctx, executionTarget.GetResourceGroup(), resourceGroup, nil) + _, err = client.CreateOrUpdate(ctx, rgName, resourceGroup, nil) if err != nil { return fmt.Errorf("failed to create resource group: %w", err) } @@ -92,7 +114,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex patchResourceGroup := armresources.ResourceGroupPatchable{ Tags: tags, } - _, err = client.Update(ctx, executionTarget.GetResourceGroup(), patchResourceGroup, nil) + _, err = client.Update(ctx, rgName, patchResourceGroup, nil) if err != nil { return fmt.Errorf("failed to update resource group: %w", err) } diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index e2a5c9c7c..0ae546d1d 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -46,6 +46,30 @@ type PipelineRunOptions struct { SubsciptionLookupFunc subsciptionLookup } +type armOutput map[string]any + +type output interface { + GetValue(key string) (*outPutValue, error) +} + +type outPutValue struct { + Type string `yaml:"type"` + Value any `yaml:"value"` +} + +func (o armOutput) GetValue(key string) (*outPutValue, error) { + if v, ok := o[key]; ok { + if innerValue, innerConversionOk := v.(map[string]any); innerConversionOk { + returnValue := outPutValue{ + Type: innerValue["type"].(string), + Value: innerValue["value"], + } + return &returnValue, nil + } + } + return nil, fmt.Errorf("key %q not found", key) +} + func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { logger := logr.FromContextOrDiscard(ctx) @@ -94,6 +118,8 @@ func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, executionTarget ExecutionTarget) error { logger := logr.FromContextOrDiscard(ctx) + outPuts := make(map[string]output) + kubeconfigFile, err := executionTarget.KubeConfig(ctx) if kubeconfigFile != "" { defer func() { @@ -107,7 +133,7 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e for _, step := range rg.Steps { // execute - err := step.run( + output, err := step.run( logr.NewContext( ctx, logger.WithValues( @@ -119,18 +145,22 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e ), kubeconfigFile, executionTarget, options, + outPuts, ) if err != nil { return err } + if output != nil { + outPuts[step.Name] = output + } } return nil } -func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions, outPuts map[string]output) (output, error) { if options.Step != "" && s.Name != options.Step { // skip steps that don't match the specified step name - return nil + return nil, nil } fmt.Println("\n---------------------") if options.DryRun { @@ -141,11 +171,22 @@ func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget E switch s.Action { case "Shell": - return s.runShellStep(ctx, kubeconfigFile, options) + return nil, s.runShellStep(ctx, kubeconfigFile, options, outPuts) case "ARM": - return s.runArmStep(ctx, executionTarget, options) + a := newArmClient(executionTarget.GetSubscriptionID(), executionTarget.GetRegion()) + if a == nil { + return nil, fmt.Errorf("failed to create ARM client") + } + output, err := a.runArmStep(ctx, options, s.Name, executionTarget.GetResourceGroup(), s.Parameters, outPuts) + if err != nil { + return nil, fmt.Errorf("failed to run ARM step: %w", err) + } + if output != nil { + return output, nil + } + return nil, nil default: - return fmt.Errorf("unsupported action type %q", s.Action) + return nil, fmt.Errorf("unsupported action type %q", s.Action) } } diff --git a/tooling/templatize/pkg/pipeline/run_test.go b/tooling/templatize/pkg/pipeline/run_test.go index f5ddde872..bb7e90c39 100644 --- a/tooling/templatize/pkg/pipeline/run_test.go +++ b/tooling/templatize/pkg/pipeline/run_test.go @@ -17,7 +17,7 @@ func TestStepRun(t *testing.T) { fooundOutput = output }, } - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}, nil) assert.NilError(t, err) assert.Equal(t, fooundOutput, "hello\n") } @@ -27,11 +27,11 @@ func TestStepRunSkip(t *testing.T) { Name: "step", } // this should skip - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}, nil) assert.NilError(t, err) // this should fail - err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}) + _, err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}, nil) assert.Error(t, err, "unsupported action type \"\"") } @@ -185,3 +185,17 @@ func TestPipelineRun(t *testing.T) { assert.NilError(t, err) assert.Equal(t, foundOutput, "hello\n") } + +func TestArmGetValue(t *testing.T) { + output := armOutput{ + "zoneName": map[string]any{ + "type": "String", + "value": "test", + }, + } + + value, err := output.GetValue("zoneName") + assert.Equal(t, value.Value, "test") + assert.Equal(t, value.Type, "String") + assert.NilError(t, err) +} diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go index e77cb788f..9e48dab90 100644 --- a/tooling/templatize/pkg/pipeline/shell.go +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -3,6 +3,7 @@ package pipeline import ( "context" "fmt" + "log" "maps" "os/exec" @@ -33,7 +34,7 @@ func (s *Step) createCommand(ctx context.Context, dryRun bool, envVars map[strin return cmd, false } -func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { +func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions, inputs map[string]output) error { if s.outputFunc == nil { s.outputFunc = func(output string) { fmt.Println(output) @@ -51,6 +52,7 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options envVars := utils.GetOsVariable() maps.Copy(envVars, stepVars) + maps.Copy(envVars, s.addInputVars(inputs)) // execute the command cmd, skipCommand := s.createCommand(ctx, options.DryRun, envVars) if skipCommand { @@ -73,6 +75,20 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options return nil } +func (s *Step) addInputVars(inputs map[string]output) map[string]string { + envVars := make(map[string]string) + for _, i := range s.Inputs { + if v, found := inputs[i.Step]; found { + value, err := v.GetValue(i.Output) + if err != nil { + log.Fatal(err) + } + envVars[i.Name] = utils.AnyToString(value.Value) + } + } + return envVars +} + func (s *Step) mapStepVariables(vars config.Variables) (map[string]string, error) { envVars := make(map[string]string) for _, e := range s.Env { diff --git a/tooling/templatize/pkg/pipeline/shell_test.go b/tooling/templatize/pkg/pipeline/shell_test.go index 078ff2255..0d8db2eee 100644 --- a/tooling/templatize/pkg/pipeline/shell_test.go +++ b/tooling/templatize/pkg/pipeline/shell_test.go @@ -163,6 +163,29 @@ func TestRunShellStep(t *testing.T) { assert.Equal(t, output, expectedOutput) }, } - err := s.runShellStep(context.Background(), "", &PipelineRunOptions{}) + err := s.runShellStep(context.Background(), "", &PipelineRunOptions{}, map[string]output{}) assert.NilError(t, err) } + +func TestAddInputVars(t *testing.T) { + mapOutput := map[string]output{} + mapOutput["step1"] = armOutput{ + "output1": map[string]any{ + "type": "String", + "value": "value1", + }, + } + s := &Step{ + Name: "step2", + Inputs: []Input{ + { + Name: "input1", + Step: "step1", + Output: "output1", + }, + }, + } + + envVars := s.addInputVars(mapOutput) + assert.DeepEqual(t, envVars, map[string]string{"input1": "value1"}) +} diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go index ec66d83bf..21d95e835 100644 --- a/tooling/templatize/pkg/pipeline/types.go +++ b/tooling/templatize/pkg/pipeline/types.go @@ -1,6 +1,8 @@ package pipeline -import "context" +import ( + "context" +) type subsciptionLookup func(context.Context, string) (string, error) @@ -48,4 +50,5 @@ type Input struct { Name string `yaml:"name"` Step string `yaml:"step"` Output string `yaml:"output"` + Type string `yaml:"type,omitempty"` }