Skip to content

Commit

Permalink
Add Output handling
Browse files Browse the repository at this point in the history 
Makes it possible to pass output of ARM steps into Shell steps as environment variables.
  • Loading branch information
@janboll
janboll committed Dec 3, 2024
1 parent a40992b commit 09174a2
Show file tree
Hide file tree
Showing 7 changed files with 237 additions and 35 deletions.
83 changes: 83 additions & 0 deletions tooling/templatize/internal/end2end/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,3 +143,86 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
_, err = rgDelResponse.PollUntilDone(context.Background(), nil)
assert.NilError(t, err)
}

func TestE2EShell(t *testing.T) {
if !shouldRunE2E() {
t.Skip("Skipping end-to-end tests")
}

tmpDir := t.TempDir()

e2eImpl := newE2E(tmpDir)

e2eImpl.AddStep(pipeline.Step{
Name: "readInput",
Action: "Shell",
Command: []string{"/bin/sh", "-c", "/usr/bin/echo ${PWD} > env.txt"},
})

persistAndRun(t, &e2eImpl)

io, err := os.ReadFile(tmpDir + "/env.txt")
assert.NilError(t, err)
assert.Equal(t, string(io), tmpDir+"\n")
}

func TestE2EArmDeployWithOutput(t *testing.T) {
if !shouldRunE2E() {
t.Skip("Skipping end-to-end tests")
}

tmpDir := t.TempDir()

e2eImpl := newE2E(tmpDir)
e2eImpl.AddStep(pipeline.Step{
Name: "createZone",
Action: "ARM",
Template: "test.bicep",
Parameters: "test.bicepparm",
})

e2eImpl.AddStep(pipeline.Step{
Name: "readInput",
Action: "Shell",
Command: []string{"/bin/sh", "-c", "echo ${zoneName} > env.txt"},
Inputs: []pipeline.Input{
{
Name: "zoneName",
Step: "createZone",
Output: "zoneName",
Type: "string",
},
},
})

e2eImpl.UseRandomRG()

e2eImpl.bicepFile = `
param zoneName string
output zoneName string = zoneName`
e2eImpl.paramFile = `
using 'test.bicep'
param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
`

persistAndRun(t, &e2eImpl)

io, err := os.ReadFile(tmpDir + "/env.txt")
assert.NilError(t, err)
assert.Equal(t, string(io), "e2etestarmdeploy.foo.bar.example.com\n")

subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)")
assert.NilError(t, err)

cred, err := azidentity.NewDefaultAzureCredential(nil)
assert.NilError(t, err)

rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil)
assert.NilError(t, err)

rgDelResponse, err := rgClient.BeginDelete(context.Background(), e2eImpl.rgName, nil)
assert.NilError(t, err)

_, err = rgDelResponse.PollUntilDone(context.Background(), nil)
assert.NilError(t, err)
}
68 changes: 45 additions & 23 deletions tooling/templatize/pkg/pipeline/arm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,64 +10,86 @@ import (
"github.com/go-logr/logr"
)

func (s *Step) runArmStep(ctx context.Context, executionTarget ExecutionTarget, options *PipelineRunOptions) error {
type armClient struct {
creds *azidentity.DefaultAzureCredential
SubscriptionID string
Region string
}

func newArmClient(subscriptionID, region string) *armClient {
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil
}
return &armClient{
creds: cred,
SubscriptionID: subscriptionID,
Region: region,
}
}

func (a *armClient) runArmStep(ctx context.Context, options *PipelineRunOptions, deploymentName string, rgName string, paramterFile string, input map[string]output) (output, error) {
logger := logr.FromContextOrDiscard(ctx)

// Transform Bicep to ARM
deploymentProperties, err := transformBicepToARM(ctx, s.Parameters, options.Vars)
deploymentProperties, err := transformBicepToARM(ctx, paramterFile, options.Vars)
if err != nil {
return fmt.Errorf("failed to transform Bicep to ARM: %w", err)
return nil, fmt.Errorf("failed to transform Bicep to ARM: %w", err)
}

// Create the deployment
deploymentName := s.Name
deployment := armresources.Deployment{
Properties: deploymentProperties,
}

// Ensure resourcegroup exists
err = s.ensureResourceGroupExists(ctx, executionTarget)
err = a.ensureResourceGroupExists(ctx, rgName)
if err != nil {
return fmt.Errorf("failed to ensure resource group exists: %w", err)
return nil, fmt.Errorf("failed to ensure resource group exists: %w", err)
}

// TODO handle dry-run

// Run deployment
cred, err := azidentity.NewDefaultAzureCredential(nil)
client, err := armresources.NewDeploymentsClient(a.SubscriptionID, a.creds, nil)
if err != nil {
return fmt.Errorf("failed to obtain a credential: %w", err)
}

client, err := armresources.NewDeploymentsClient(executionTarget.GetSubscriptionID(), cred, nil)
if err != nil {
return fmt.Errorf("failed to create deployments client: %w", err)
return nil, fmt.Errorf("failed to create deployments client: %w", err)
}

poller, err := client.BeginCreateOrUpdate(ctx, executionTarget.GetResourceGroup(), deploymentName, deployment, nil)
poller, err := client.BeginCreateOrUpdate(ctx, rgName, deploymentName, deployment, nil)
if err != nil {
return fmt.Errorf("failed to create deployment: %w", err)
return nil, fmt.Errorf("failed to create deployment: %w", err)
}
logger.Info("Deployment started", "deployment", deploymentName)

// Wait for completion
resp, err := poller.PollUntilDone(ctx, nil)
if err != nil {
return fmt.Errorf("failed to wait for deployment completion: %w", err)
return nil, fmt.Errorf("failed to wait for deployment completion: %w", err)
}
logger.Info("Deployment finished successfully", "deployment", deploymentName, "responseId", *resp.ID)
return nil

if resp.Properties.Outputs != nil {
if outputMap, ok := resp.Properties.Outputs.(map[string]any); ok {
returnMap := armOutput{}
for k, v := range outputMap {
returnMap[k] = v
}
return returnMap, nil
}
}
return nil, nil
}

func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget ExecutionTarget) error {
func (a *armClient) ensureResourceGroupExists(ctx context.Context, rgName string) error {
// Create a new Azure identity client
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return fmt.Errorf("failed to obtain a credential: %w", err)
}

// Create a new ARM client
client, err := armresources.NewResourceGroupsClient(executionTarget.GetSubscriptionID(), cred, nil)
client, err := armresources.NewResourceGroupsClient(a.SubscriptionID, cred, nil)
if err != nil {
return fmt.Errorf("failed to create ARM client: %w", err)
}
Expand All @@ -77,22 +99,22 @@ func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget Ex
tags := map[string]*string{
"persist": to.Ptr("true"),
}
_, err = client.Get(ctx, executionTarget.GetResourceGroup(), nil)
_, err = client.Get(ctx, rgName, nil)
if err != nil {
// Create the resource group
resourceGroup := armresources.ResourceGroup{
Location: to.Ptr(executionTarget.GetRegion()),
Location: to.Ptr(a.Region),
Tags: tags,
}
_, err = client.CreateOrUpdate(ctx, executionTarget.GetResourceGroup(), resourceGroup, nil)
_, err = client.CreateOrUpdate(ctx, rgName, resourceGroup, nil)
if err != nil {
return fmt.Errorf("failed to create resource group: %w", err)
}
} else {
patchResourceGroup := armresources.ResourceGroupPatchable{
Tags: tags,
}
_, err = client.Update(ctx, executionTarget.GetResourceGroup(), patchResourceGroup, nil)
_, err = client.Update(ctx, rgName, patchResourceGroup, nil)
if err != nil {
return fmt.Errorf("failed to update resource group: %w", err)
}
Expand Down
53 changes: 47 additions & 6 deletions tooling/templatize/pkg/pipeline/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,30 @@ type PipelineRunOptions struct {
SubsciptionLookupFunc subsciptionLookup
}

type armOutput map[string]any

type output interface {
GetValue(key string) (*outPutValue, error)
}

type outPutValue struct {
Type string `yaml:"type"`
Value any `yaml:"value"`
}

func (o armOutput) GetValue(key string) (*outPutValue, error) {
if v, ok := o[key]; ok {
if innerValue, innerConversionOk := v.(map[string]any); innerConversionOk {
returnValue := outPutValue{
Type: innerValue["type"].(string),
Value: innerValue["value"],
}
return &returnValue, nil
}
}
return nil, fmt.Errorf("key %q not found", key)
}

func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error {
logger := logr.FromContextOrDiscard(ctx)

Expand Down Expand Up @@ -94,6 +118,8 @@ func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error {
func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, executionTarget ExecutionTarget) error {
logger := logr.FromContextOrDiscard(ctx)

outPuts := make(map[string]output)

kubeconfigFile, err := executionTarget.KubeConfig(ctx)
if kubeconfigFile != "" {
defer func() {
Expand All @@ -107,7 +133,7 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e

for _, step := range rg.Steps {
// execute
err := step.run(
output, err := step.run(
logr.NewContext(
ctx,
logger.WithValues(
Expand All @@ -119,18 +145,22 @@ func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions, e
),
kubeconfigFile,
executionTarget, options,
outPuts,
)
if err != nil {
return err
}
if output != nil {
outPuts[step.Name] = output
}
}
return nil
}

func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions) error {
func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget ExecutionTarget, options *PipelineRunOptions, outPuts map[string]output) (output, error) {
if options.Step != "" && s.Name != options.Step {
// skip steps that don't match the specified step name
return nil
return nil, nil
}
fmt.Println("\n---------------------")
if options.DryRun {
Expand All @@ -141,11 +171,22 @@ func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget E

switch s.Action {
case "Shell":
return s.runShellStep(ctx, kubeconfigFile, options)
return nil, s.runShellStep(ctx, kubeconfigFile, options, outPuts)
case "ARM":
return s.runArmStep(ctx, executionTarget, options)
a := newArmClient(executionTarget.GetSubscriptionID(), executionTarget.GetRegion())
if a == nil {
return nil, fmt.Errorf("failed to create ARM client")
}
output, err := a.runArmStep(ctx, options, s.Name, executionTarget.GetResourceGroup(), s.Parameters, outPuts)
if err != nil {
return nil, fmt.Errorf("failed to run ARM step: %w", err)
}
if output != nil {
return output, nil
}
return nil, nil
default:
return fmt.Errorf("unsupported action type %q", s.Action)
return nil, fmt.Errorf("unsupported action type %q", s.Action)
}
}

Expand Down
20 changes: 17 additions & 3 deletions tooling/templatize/pkg/pipeline/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ func TestStepRun(t *testing.T) {
fooundOutput = output
},
}
err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{})
_, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{}, nil)
assert.NilError(t, err)
assert.Equal(t, fooundOutput, "hello\n")
}
Expand All @@ -27,11 +27,11 @@ func TestStepRunSkip(t *testing.T) {
Name: "step",
}
// this should skip
err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"})
_, err := s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "skip"}, nil)
assert.NilError(t, err)

// this should fail
err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"})
_, err = s.run(context.Background(), "", &executionTargetImpl{}, &PipelineRunOptions{Step: "step"}, nil)
assert.Error(t, err, "unsupported action type \"\"")
}

Expand Down Expand Up @@ -185,3 +185,17 @@ func TestPipelineRun(t *testing.T) {
assert.NilError(t, err)
assert.Equal(t, foundOutput, "hello\n")
}

func TestArmGetValue(t *testing.T) {
output := armOutput{
"zoneName": map[string]any{
"type": "String",
"value": "test",
},
}

value, err := output.GetValue("zoneName")
assert.Equal(t, value.Value, "test")
assert.Equal(t, value.Type, "String")
assert.NilError(t, err)
}
Loading

0 comments on commit 09174a2

Please sign in to comment.