diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go index 30482ab7b..581a10943 100644 --- a/tooling/templatize/internal/end2end/e2e_test.go +++ b/tooling/templatize/internal/end2end/e2e_test.go @@ -143,3 +143,55 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com' _, err = rgDelResponse.PollUntilDone(context.Background(), nil) assert.NilError(t, err) } + +func TestE2EArmDeployWithOutput(t *testing.T) { + // if !shouldRunE2E() { + // t.Skip("Skipping end-to-end tests") + // } + + tmpDir := t.TempDir() + + e2eImpl := newE2E(tmpDir) + e2eImpl.AddStep(pipeline.Step{ + Name: "createZone", + Action: "ARM", + Template: "test.bicep", + Parameters: "test.bicepparm", + }) + + e2eImpl.AddStep(pipeline.Step{ + Name: "readInput", + Action: "Shell", + Command: []string{"/usr/bin/env"}, + Inputs: []pipeline.Input{ + { + Name: "zoneName", + Step: "createZone", + Output: "zoneName", + Type: "string", + }, + }, + }) + + e2eImpl.UseRandomRG() + + e2eImpl.bicepFile = ` +param zoneName string +resource symbolicname 'Microsoft.Network/dnsZones@2018-05-01' = { + location: 'global' + name: zoneName +} + +output zoneName string = symbolicname.name` + e2eImpl.paramFile = ` +using 'test.bicep' +param zoneName = 'e2etestarmdeploy.foo.bar.example.com' +` + + persistAndRun(t, &e2eImpl) + + io, err := os.ReadFile(tmpDir + "/env.txt") + assert.NilError(t, err) + assert.Equal(t, string(io), "test_env\n") + +} diff --git a/tooling/templatize/pkg/pipeline/arm.go b/tooling/templatize/pkg/pipeline/arm.go index 888bce862..4a450c377 100644 --- a/tooling/templatize/pkg/pipeline/arm.go +++ b/tooling/templatize/pkg/pipeline/arm.go @@ -10,56 +10,73 @@ import ( "github.com/go-logr/logr" ) -func (s *Step) runArmStep(ctx context.Context, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +type armClient struct { + creds *azidentity.DefaultAzureCredential + deployClient *armresources.DeploymentsClient + SubscriptionID string + Region string +} + +func newArmClient(subscriptionID, region string) *armClient { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil + } + return &armClient{ + creds: cred, + SubscriptionID: subscriptionID, + Region: region, + } +} + +func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, deploymentName string, rgName string, paramterFile string, input map[string]outputImpl) (any, error) { logger := logr.FromContextOrDiscard(ctx) // Transform Bicep to ARM - deploymentProperties, err := transformBicepToARM(ctx, s.Parameters, options.Vars) + deploymentProperties, err := transformBicepToARM(ctx, paramterFile, options.Vars) if err != nil { - return fmt.Errorf("failed to transform Bicep to ARM: %w", err) + return nil, fmt.Errorf("failed to transform Bicep to ARM: %w", err) } // Create the deployment - deploymentName := s.Name deployment := armresources.Deployment{ Properties: deploymentProperties, } // Ensure resourcegroup exists - err = s.ensureResourceGroupExists(ctx, executionTarget) + err = a.ensureResourceGroupExists(ctx, rgName) if err != nil { - return fmt.Errorf("failed to ensure resource group exists: %w", err) + return nil, fmt.Errorf("failed to ensure resource group exists: %w", err) } // TODO handle dry-run // Run deployment - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return fmt.Errorf("failed to obtain a credential: %w", err) - } - - client, err := armresources.NewDeploymentsClient(executionTarget.GetSubscriptionID(), cred, nil) + client, err := armresources.NewDeploymentsClient(a.SubscriptionID, a.creds, nil) if err != nil { - return fmt.Errorf("failed to create deployments client: %w", err) + return nil, fmt.Errorf("failed to create deployments client: %w", err) } - poller, err := client.BeginCreateOrUpdate(ctx, executionTarget.GetResourceGroup(), deploymentName, deployment, nil) + poller, err := client.BeginCreateOrUpdate(ctx, rgName, deploymentName, deployment, nil) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + return nil, fmt.Errorf("failed to create deployment: %w", err) } logger.Info("Deployment started", "deployment", deploymentName) // Wait for completion resp, err := poller.PollUntilDone(ctx, nil) if err != nil { - return fmt.Errorf("failed to wait for deployment completion: %w", err) + return nil, fmt.Errorf("failed to wait for deployment completion: %w", err) } logger.Info("Deployment finished successfully", "deployment", deploymentName, "responseId", *resp.ID) - return nil + + if resp.Properties.Outputs != nil { + return resp.Properties.Outputs, nil + } + return nil, nil } -func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget ExecutionTarget) error { +func (a *armClient) ensureResourceGroupExists(ctx context.Context, rgName string) error { // Create a new Azure identity client cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -67,7 +84,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex } // Create a new ARM client - client, err := armresources.NewResourceGroupsClient(executionTarget.GetSubscriptionID(), cred, nil) + client, err := armresources.NewResourceGroupsClient(a.SubscriptionID, cred, nil) if err != nil { return fmt.Errorf("failed to create ARM client: %w", err) } @@ -77,14 +94,14 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex tags := map[string]*string{ "persist": to.Ptr("true"), } - _, err = client.Get(ctx, executionTarget.GetResourceGroup(), nil) + _, err = client.Get(ctx, rgName, nil) if err != nil { // Create the resource group resourceGroup := armresources.ResourceGroup{ - Location: to.Ptr(executionTarget.GetRegion()), + Location: to.Ptr(a.Region), Tags: tags, } - _, err = client.CreateOrUpdate(ctx, executionTarget.GetResourceGroup(), resourceGroup, nil) + _, err = client.CreateOrUpdate(ctx, rgName, resourceGroup, nil) if err != nil { return fmt.Errorf("failed to create resource group: %w", err) } @@ -92,7 +109,7 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex patchResourceGroup := armresources.ResourceGroupPatchable{ Tags: tags, } - _, err = client.Update(ctx, executionTarget.GetResourceGroup(), patchResourceGroup, nil) + _, err = client.Update(ctx, rgName, patchResourceGroup, nil) if err != nil { return fmt.Errorf("failed to update resource group: %w", err) } diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index e2a5c9c7c..06980325c 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -46,6 +46,33 @@ type PipelineRunOptions struct { SubsciptionLookupFunc subsciptionLookup } +type outputImpl map[string]any + +type output interface { + GetString(key string) (string, error) + GetNumber(key string) (int, error) +} + +func (o outputImpl) GetString(stepRef, key string) (string, error) { + if stepOutput, ok := o[stepRef]; !ok { + return "", fmt.Errorf("step output %q not found", stepRef) + } else if stepOutputAsMap, conversionOk := stepOutput.(map[string]any); conversionOk { + if v, ok := stepOutputAsMap[key]; ok { + + if innerValue, innerConversionOk := v.(map[string]any); innerConversionOk { + return fmt.Sprintf("%v", innerValue["value"]), nil + } else { + return "", fmt.Errorf("key %q not found", key) + } + } + } + return "", fmt.Errorf("key %q not found", key) +} + +func GetNumber(key string) (int, error) { + return 0, nil +} + func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { logger := logr.FromContextOrDiscard(ctx) @@ -94,6 +121,8 @@ func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, executionTarget ExecutionTarget) error { logger := logr.FromContextOrDiscard(ctx) + outPuts := make(map[string]outputImpl) + kubeconfigFile, err := executionTarget.KubeConfig(ctx) if kubeconfigFile != "" { defer func() { @@ -107,7 +136,7 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e for _, step := range rg.Steps { // execute - err := step.run( + output, err := step.run( logr.NewContext( ctx, logger.WithValues( @@ -119,18 +148,22 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e ), kubeconfigFile, executionTarget, options, + outPuts, ) if err != nil { return err } + if output != nil { + outPuts[step.Name] = outputImpl(output) + } } return nil } -func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions) error { +func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions, outPuts map[string]outputImpl) (outputImpl, error) { if options.Step != "" && s.Name != options.Step { // skip steps that don't match the specified step name - return nil + return nil, nil } fmt.Println("\n---------------------") if options.DryRun { @@ -141,11 +174,22 @@ func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget E switch s.Action { case "Shell": - return s.runShellStep(ctx, kubeconfigFile, options) + return nil, s.runShellStep(ctx, kubeconfigFile, options, outPuts) case "ARM": - return s.runArmStep(ctx, executionTarget, options) + a := newArmClient(executionTarget.GetSubscriptionID(), executionTarget.GetRegion()) + if a == nil { + return nil, fmt.Errorf("failed to create ARM client") + } + output, err := a.runArmStep(ctx, options, s.Name, executionTarget.GetResourceGroup(), s.Parameters, outPuts) + if err != nil { + return nil, fmt.Errorf("failed to run ARM step: %w", err) + } + if output != nil { + return outputImpl{s.Name: output}, nil + } + return nil, nil default: - return fmt.Errorf("unsupported action type %q", s.Action) + return nil, fmt.Errorf("unsupported action type %q", s.Action) } } diff --git a/tooling/templatize/pkg/pipeline/run_test.go b/tooling/templatize/pkg/pipeline/run_test.go index f5ddde872..caa51152d 100644 --- a/tooling/templatize/pkg/pipeline/run_test.go +++ b/tooling/templatize/pkg/pipeline/run_test.go @@ -17,7 +17,7 @@ func TestStepRun(t *testing.T) { fooundOutput = output }, } - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}, nil) assert.NilError(t, err) assert.Equal(t, fooundOutput, "hello\n") } @@ -27,11 +27,11 @@ func TestStepRunSkip(t *testing.T) { Name: "step", } // this should skip - err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}) + _, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}, nil) assert.NilError(t, err) // this should fail - err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}) + _, err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}, nil) assert.Error(t, err, "unsupported action type \"\"") } diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go index e77cb788f..c2509cd6a 100644 --- a/tooling/templatize/pkg/pipeline/shell.go +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -3,6 +3,7 @@ package pipeline import ( "context" "fmt" + "log" "maps" "os/exec" @@ -33,7 +34,7 @@ func (s *Step) createCommand(ctx context.Context, dryRun bool, envVars map[strin return cmd, false } -func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { +func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions, inputs map[string]outputImpl) error { if s.outputFunc == nil { s.outputFunc = func(output string) { fmt.Println(output) @@ -51,6 +52,7 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options envVars := utils.GetOsVariable() maps.Copy(envVars, stepVars) + maps.Copy(envVars, s.addInputVars(inputs)) // execute the command cmd, skipCommand := s.createCommand(ctx, options.DryRun, envVars) if skipCommand { @@ -73,6 +75,20 @@ func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options return nil } +func (s *Step) addInputVars(inputs map[string]outputImpl) map[string]string { + envVars := make(map[string]string) + for _, i := range s.Inputs { + if v, found := inputs[i.Step]; found { + value, err := v.GetString(i.Step, i.Output) + if err != nil { + log.Fatal(err) + } + envVars[i.Name] = value + } + } + return envVars +} + func (s *Step) mapStepVariables(vars config.Variables) (map[string]string, error) { envVars := make(map[string]string) for _, e := range s.Env { diff --git a/tooling/templatize/pkg/pipeline/shell_test.go b/tooling/templatize/pkg/pipeline/shell_test.go index 078ff2255..8e1ee5908 100644 --- a/tooling/templatize/pkg/pipeline/shell_test.go +++ b/tooling/templatize/pkg/pipeline/shell_test.go @@ -163,6 +163,26 @@ func TestRunShellStep(t *testing.T) { assert.Equal(t, output, expectedOutput) }, } - err := s.runShellStep(context.Background(), "", &PipelineRunOptions{}) + err := s.runShellStep(context.Background(), "", &PipelineRunOptions{}, map[string]outputImpl{}) assert.NilError(t, err) } + +func TestAddInputVars(t *testing.T) { + mapOutput := map[string]outputImpl{ + "step1": { + "output1": "value1", + }, + } + s := &Step{ + Inputs: []Input{ + { + Name: "input1", + Step: "step1", + Output: "output1", + }, + }, + } + + envVars := s.addInputVars(mapOutput) + assert.DeepEqual(t, envVars, map[string]string{"input1": "value1"}) +} diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go index ec66d83bf..21d95e835 100644 --- a/tooling/templatize/pkg/pipeline/types.go +++ b/tooling/templatize/pkg/pipeline/types.go @@ -1,6 +1,8 @@ package pipeline -import "context" +import ( + "context" +) type subsciptionLookup func(context.Context, string) (string, error) @@ -48,4 +50,5 @@ type Input struct { Name string `yaml:"name"` Step string `yaml:"step"` Output string `yaml:"output"` + Type string `yaml:"type,omitempty"` }