Merge pull request #910 from Azure/add-kubernetes-e2e

Add an e2e for Kubernetes access

.github/workflows/ci-go.yml

@@ -12,9 +12,24 @@ on:
 jobs:
   test:
     permissions:
+      id-token: 'write'
       contents: 'read'
     runs-on: 'ubuntu-latest'
     steps:
+      - name: "install azure-cli"
+        uses: "Azure/ARO-HCP@main"
+
+      - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2
+        with:
+          kubelogin-version: 'v0.1.3'
+
+      - name: 'Az CLI login'
+        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
+        with:
+            client-id: ${{ secrets.AZURE_CLIENT_ID }}
+            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
@@ -26,7 +41,8 @@ jobs:
           check-latest: true
 
       - name: 'Test'
-        run: make test
+        run: PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make test
+
   lint:
     permissions:
       contents: 'read'

Makefile

@@ -15,7 +15,7 @@ all: test lint
 # There is currently no convenient way to run tests against a whole Go workspace
 # https://github.com/golang/go/issues/50745
 test:
-	RUN_TEMPLATIZE_E2E=true go list -f '{{.Dir}}/...' -m | xargs go test -tags=$(GOTAGS) -cover
+	go list -f '{{.Dir}}/...' -m |RUN_TEMPLATIZE_E2E=true xargs go test -tags=$(GOTAGS) -cover
 .PHONY: test
 
 # There is currently no convenient way to run golangci-lint against a whole Go workspace

tooling/templatize/internal/end2end/e2e.go

@@ -1,10 +1,12 @@
 package testutil
 
 import (
+	"fmt"
 	"os"
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
@@ -13,61 +15,80 @@ func shouldRunE2E() bool {
 }
 
 type E2E interface {
-	Pipeline(step pipeline.Step, aksName string) error
+	SetConfig(updates config.Variables)
+	AddStep(step pipeline.Step)
+	SetOSArgs()
 	Persist() error
 }
 
 type e2eImpl struct {
-	config   string
+	config   config.Variables
 	makefile string
-	pipeline string
+	pipeline pipeline.Pipeline
 	schema   string
 	tmpdir   string
 }
 
+var _ E2E = &e2eImpl{}
+
 func newE2E(tmpdir string) e2eImpl {
-	return e2eImpl{
+	imp := e2eImpl{
 		tmpdir: tmpdir,
 		schema: `{"type": "object"}`,
-		config: `
-$schema: schema.json
-defaults:
-    region: {{ .ctx.region }}
-    subscription: ARO Hosted Control Planes (EA Subscription 1)
-    rg: hcp-templatize
-    test_env: test_env
-clouds:
-    public:
-        defaults:
-        environments:
-            dev:
-                defaults:
-`}
-}
-
-func (e *e2eImpl) SetPipeline(step pipeline.Step, aksName string) error {
-	p := pipeline.Pipeline{
-		ServiceGroup: "Microsoft.Azure.ARO.Test",
-		RolloutName:  "Test Rollout",
-		ResourceGroups: []*pipeline.ResourceGroup{
-			{
-				Name:         "{{ .rg }}",
-				Subscription: "{{ .subscription }}",
-				Steps: []*pipeline.Step{
-					&step,
+		config: config.Variables{
+			"$schema": "schema.json",
+			"defaults": config.Variables{
+				"region":       "westus3",
+				"subscription": "ARO Hosted Control Planes (EA Subscription 1)",
+				"rg":           "hcp-templatize",
+				"test_env":     "test_env",
+			},
+			"clouds": config.Variables{
+				"public": config.Variables{
+					"defaults": config.Variables{},
+					"environments": config.Variables{
+						"dev": config.Variables{
+							"defaults": config.Variables{},
+						},
+					},
+				},
+			},
+		},
+		pipeline: pipeline.Pipeline{
+			ServiceGroup: "Microsoft.Azure.ARO.Test",
+			RolloutName:  "Test Rollout",
+			ResourceGroups: []*pipeline.ResourceGroup{
+				{
+					Name:         "{{ .rg }}",
+					Subscription: "{{ .subscription }}",
 				},
 			},
 		},
 	}
-	if aksName != "" {
-		p.ResourceGroups[0].AKSCluster = aksName
-	}
-	out, err := yaml.Marshal(p)
-	if err != nil {
-		return err
+
+	imp.SetOSArgs()
+	return imp
+}
+
+func (e *e2eImpl) SetOSArgs() {
+	os.Args = []string{"test",
+		"--cloud", "public",
+		"--pipeline-file", e.tmpdir + "/pipeline.yaml",
+		"--config-file", e.tmpdir + "/config.yaml",
+		"--deploy-env", "dev",
 	}
-	e.pipeline = string(out)
-	return nil
+}
+
+func (e *e2eImpl) SetAKSName(aksName string) {
+	e.pipeline.ResourceGroups[0].AKSCluster = aksName
+}
+
+func (e *e2eImpl) AddStep(step pipeline.Step) {
+	e.pipeline.ResourceGroups[0].Steps = append(e.pipeline.ResourceGroups[0].Steps, &step)
+}
+
+func (e *e2eImpl) SetConfig(updates config.Variables) {
+	config.MergeVariables(e.config, updates)
 }
 
 func (e *e2eImpl) Persist() error {
@@ -77,13 +98,25 @@ func (e *e2eImpl) Persist() error {
 			return err
 		}
 	}
-	err := os.WriteFile(e.tmpdir+"/config.yaml", []byte(e.config), 0644)
+
+	configBytes, err := yaml.Marshal(e.config)
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+
+	err = os.WriteFile(e.tmpdir+"/config.yaml", configBytes, 0644)
 	if err != nil {
 		return err
 	}
+
 	err = os.WriteFile(e.tmpdir+"/schema.json", []byte(e.schema), 0644)
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(e.tmpdir+"/pipeline.yaml", []byte(e.pipeline), 0644)
+
+	pipelineBytes, err := yaml.Marshal(e.pipeline)
+	if err != nil {
+		return fmt.Errorf("failed to marshal pipeline: %w", err)
+	}
+	return os.WriteFile(e.tmpdir+"/pipeline.yaml", []byte(pipelineBytes), 0644)
 }

tooling/templatize/internal/end2end/e2e_test.go

@@ -7,6 +7,7 @@ import (
 	"gotest.tools/v3/assert"
 
 	"github.com/Azure/ARO-HCP/tooling/templatize/cmd/pipeline/run"
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
@@ -18,7 +19,7 @@ func TestE2EMake(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	e2eImpl := newE2E(tmpDir)
-	err := e2eImpl.SetPipeline(pipeline.Step{
+	e2eImpl.AddStep(pipeline.Step{
 		Name:    "test",
 		Action:  "Shell",
 		Command: []string{"make", "test"},
@@ -28,35 +29,54 @@ func TestE2EMake(t *testing.T) {
 				ConfigRef: "test_env",
 			},
 		},
-	}, "")
-	assert.NilError(t, err)
+	})
 
 	e2eImpl.makefile = `
 test:
 	echo ${TEST_ENV} > env.txt
 `
-	err = e2eImpl.Persist()
+	err := e2eImpl.Persist()
 	assert.NilError(t, err)
 
 	cmd, err := run.NewCommand()
 
 	assert.NilError(t, err)
 
-	os.Args = []string{"test",
-		"--cloud", "public",
-		"--pipeline-file", tmpDir + "/pipeline.yaml",
-		"--step", "test",
-		"--config-file", tmpDir + "/config.yaml",
-		"--deploy-env", "dev",
+	err = cmd.Execute()
+	assert.NilError(t, err)
+
+	fno, err := os.Stat(tmpDir + "/env.txt")
+	assert.NilError(t, err)
+	assert.Equal(t, fno.Size(), int64(9))
+}
+
+func TestE2EKubernetes(t *testing.T) {
+	if !shouldRunE2E() {
+		t.Skip("Skipping end-to-end tests")
 	}
 
-	err = cmd.Execute()
+	tmpDir := t.TempDir()
 
+	e2eImpl := newE2E(tmpDir)
+	e2eImpl.AddStep(pipeline.Step{
+		Name:    "test",
+		Action:  "Shell",
+		Command: []string{"kubectl", "get", "namespaces"},
+	})
+	e2eImpl.SetAKSName("aro-hcp-aks")
+
+	e2eImpl.SetConfig(config.Variables{"defaults": config.Variables{"rg": "hcp-underlay-dev-svc"}})
+
+	err := e2eImpl.Persist()
 	assert.NilError(t, err)
 
-	fno, err := os.Stat(tmpDir + "/env.txt")
+	cmd, err := run.NewCommand()
 
 	assert.NilError(t, err)
 
-	assert.Equal(t, fno.Size(), int64(9))
+	e2eImpl.SetOSArgs()
+
+	err = cmd.Execute()
+	assert.NilError(t, err)
+
 }

tooling/templatize/pkg/aks/admin.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -92,6 +93,11 @@ func CheckClusterAdminPermissions(ctx context.Context, kubeconfigPath string) er
 }
 
 func getCurrentUserObjectID(ctx context.Context) (string, error) {
+
+	if os.Getenv("PRINCIPAL_ID") != "" {
+		return os.Getenv("PRINCIPAL_ID"), nil
+	}
+
 	// Create a Graph client using Azure Credentials
 	cred, err := azidentity.NewDefaultAzureCredential(nil)
 	if err != nil {

tooling/templatize/pkg/config/config.go

@@ -76,13 +76,13 @@ func InterfaceToVariables(i interface{}) (Variables, bool) {
 // Merges variables, returns merged variables
 // However the return value is only used for recursive updates on the map
 // The actual merged variables are updated in the base map
-func mergeVariables(base, override Variables) Variables {
+func MergeVariables(base, override Variables) Variables {
 	for k, newValue := range override {
 		if baseValue, exists := base[k]; exists {
 			srcMap, srcMapOk := InterfaceToVariables(newValue)
 			dstMap, dstMapOk := InterfaceToVariables(baseValue)
 			if srcMapOk && dstMapOk {
-				newValue = mergeVariables(dstMap, srcMap)
+				newValue = MergeVariables(dstMap, srcMap)
 			}
 		}
 		base[k] = newValue
@@ -158,7 +158,7 @@ func (cp *configProviderImpl) GetVariables(cloud, deployEnv, region string, conf
 	if err != nil {
 		return nil, err
 	}
-	mergeVariables(variables, regionOverrides)
+	MergeVariables(variables, regionOverrides)
 
 	// validate schema
 	err = cp.validateSchema(variables)
@@ -198,9 +198,9 @@ func (cp *configProviderImpl) GetDeployEnvVariables(cloud, deployEnv string, con
 	}
 
 	variables := Variables{}
-	mergeVariables(variables, config.GetDefaults())
-	mergeVariables(variables, config.GetCloudOverrides(cloud))
-	mergeVariables(variables, config.GetDeployEnvOverrides(cloud, deployEnv))
+	MergeVariables(variables, config.GetDefaults())
+	MergeVariables(variables, config.GetCloudOverrides(cloud))
+	MergeVariables(variables, config.GetDeployEnvOverrides(cloud, deployEnv))
 
 	cp.schema = config.GetSchema()
 

tooling/templatize/pkg/config/config_test.go

@@ -151,7 +151,7 @@ func TestMergeVariable(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			result := mergeVariables(tc.base, tc.override)
+			result := MergeVariables(tc.base, tc.override)
 			assert.Equal(t, tc.expected, result)
 		})
 	}