Merge pull request #958 from shajmakh/introspect-features

e2e: manager: add features introspection

.github/workflows/e2e-tools.yml

@@ -43,7 +43,7 @@ jobs:
 
     - name: Build tools
       run: |
-        make build-tools binary-nrovalidate
+        make build-tools binary-nrovalidate bin/mkginkgolabelfilter binary
 
     - name: Create K8s Kind Cluster
       run: |

.github/workflows/support-tools.yml

@@ -0,0 +1,43 @@
+name: Build support tools binaries
+
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: setup golang
+        uses: actions/setup-go@v2
+        id: go
+        with:
+          go-version: 1.22.4
+
+  build:
+    needs: [setup]
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: build tools' binaries
+        run: |
+          make bin/mkginkgolabelfilter
+          make bin/catkubeletconfmap
+
+      - name: release the binaries
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: bin/*
+          tag: support-tools

Makefile

@@ -386,7 +386,7 @@ goversion:
 build-tools: goversion bin/buildhelper bin/envsubst bin/lsplatform
 
 .PHONY: build-tools-all
-build-tools-all: goversion bin/buildhelper bin/envsubst bin/lsplatform bin/catkubeletconfmap bin/watchnrtattr
+build-tools-all: goversion bin/buildhelper bin/envsubst bin/lsplatform bin/catkubeletconfmap bin/watchnrtattr bin/mkginkgolabelfilter
 
 bin/buildhelper:
 	@go build -o bin/buildhelper tools/buildhelper/buildhelper.go
@@ -398,11 +398,16 @@ bin/lsplatform:
 	@go build -o bin/lsplatform tools/lsplatform/lsplatform.go
 
 bin/catkubeletconfmap:
-	@go build -o bin/catkubeletconfmap tools/catkubeletconfmap/catkubeletconfmap.go
+	LDFLAGS="-static"
+	CGO_ENABLED=0 go build -o bin/catkubeletconfmap -ldflags "$$LDFLAGS" tools/catkubeletconfmap/catkubeletconfmap.go
 
 bin/watchnrtattr:
 	@go build -o bin/watchnrtattr tools/watchnrtattr/watchnrtattr.go
 
+bin/mkginkgolabelfilter:
+	LDFLAGS="-static"
+	@go build -o bin/mkginkgolabelfilter -ldflags "$$LDFLAGS" tools/mkginkgolabelfilter/mkginkgolabelfilter.go
+
 verify-generated: bundle generate
 	@echo "Verifying that all code is committed after updating deps and formatting and generating code"
 	hack/verify-generated.sh

doc/features/README.md

@@ -0,0 +1,52 @@
+Features introspection in e2e tests
+======================================
+
+**Goal**
+
+Provide a way to identify the active features for every supported operator version.
+
+**Design**
+
+- Provide the possibility to group a set of tests that verify a common feature using ginkgo Label("feature:<topic>").
+  A Topic should describe the main feature being tested or the unique effect of the test on the cluster. For instance:
+  `feature:schedrst`: indicates that the test is expected to restart the schedular, be it a pod restart or a complete removal of the scheduler and recreation.
+  `feature:rtetols`: indicates that the test is testing the new feature RTE tolerations, which involves specific non-default configuration on the operator CRs hence functional effects.
+  `feature:unsched`: indicates that the test workload is anticipated to be un-schedulable for any reason for example: insufficient resources (pod is pending), TAE (failed)..
+  `feature:wlplacement`: indicates that the test challenges the scheduler to place a workload on a node. This is a common label for most of the functional tests.
+    - Rules for creating a new topic
+        - A topic should point out briefly the new feature being tested or the unique effect of the test on the cluster
+        - The topic should be lowercase
+        - For composite topics use `_` between the words, e.g no_nrt
+        - Should not consist of any of `&|!,()/`
+- Define the list of supported features for each version.
+- Allow the user to list the supported features by adding new flag, `-inspect-features` which if passed the numaresources-operator binary on the controller pod will list the active features for the deployed version of the operator.
+- For automated bugs' scenarios that their fix is not back-ported to all versions use keywords tags that briefly describes the bug's main check.
+
+**List active features**
+
+Once the operator is installed, perform the following on the controller manager pod:
+
+```azure
+oc exec -it numaresources-controller-manager-95dd55c6-k6428 -n openshift-numaresources -- /bin/numaresources-operator --inspect-features
+ ```
+
+Example output:
+```
+{"active":["config","nonreg","hostlevel","resacct","cache","stall","rmsched","rtetols","overhead","wlplacement","unsched","nonrt","taint","nodelabel","byres","tmpol"]}
+```
+
+**Use case example: run tests for supported features only**
+
+To build the filter label query of the supported features on a specific version, a binary called `mkginkgolabelfilter` exists in `releases/support-tools` which expects a JSON input showing the supported features from the operator controller pod as follows:
+
+To use the helper tool perform the following:
+
+```
+# curl -L -o mkginkgolabelfilter https://github.com/openshift-kni/numaresources-operator/releases/download/support-tools/mkginkgolabelfilter
+# chmod 755 mkginkgolabelfilter
+# ./mkginkgolabelfilter # This will wait to read the input which should be the output of the --inspect-features above
+{"active":["config","nonreg","hostlevel","resacct","cache","stall","rmsched","rtetols","overhead","wlplacement","unsched","nonrt","taint","nodelabel","byres","tmpol"]}
+feature: consistAny {config,nonreg,hostlevel,resacct,cache,stall,rmsched,rtetols,overhead,wlplacement,unsched,nonrt,taint,nodelabel,byres,tmpol}
+```
+
+Then later in the podman command of running the tests use `--filter-label` with the output of the tool to run tests of supported features only.  

internal/api/features/_topics.json

@@ -0,0 +1,20 @@
+{
+  "active": [
+    "config",
+    "nonreg",
+    "hostlevel",
+    "resacct",
+    "cache",
+    "stall",
+    "schedrst",
+    "rtetols",
+    "overhead",
+    "wlplacement",
+    "unsched",
+    "nonrt",
+    "taint",
+    "nodelabel",
+    "byres",
+    "tmpol"
+  ]
+}

internal/api/features/topics.go

@@ -0,0 +1,31 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2024 Red Hat, Inc.
+ */
+
+package features
+
+import (
+	_ "embed"
+	"encoding/json"
+)
+
+//go:embed _topics.json
+var topicsData string
+
+func GetTopics() TopicInfo {
+	var tp TopicInfo
+	_ = json.Unmarshal([]byte(topicsData), &tp)
+	return tp
+}

internal/api/features/topics_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"strings"
+	"testing"
+
+	"k8s.io/utils/strings/slices"
+)
+
+func TestGetTopics(t *testing.T) {
+	type testcase struct {
+		name       string
+		topicsList string
+		isPositive bool
+	}
+	testcases := []testcase{
+		{
+			name:       "with list of well known topics that every release operator must support",
+			topicsList: "config,nonreg,hostlevel,resacct,cache,stall,schedrst,overhead,wlplacement,unsched,taint,nodelabel,byres,tmpol",
+			isPositive: true,
+		},
+		{
+			name:       "with inactive topics included",
+			topicsList: "config,nonreg,hostlevel,notfound,cache",
+			isPositive: false,
+		},
+	}
+
+	actual := GetTopics()
+	for _, tc := range testcases {
+		topicsStr := strings.TrimSpace(tc.topicsList)
+		topics := strings.Split(topicsStr, ",")
+		tp := findMissingTopics(actual.Active, topics)
+
+		if len(tp) > 0 && tc.isPositive {
+			t.Errorf("expected to include topic(s) %v but it didn't, list found: %v", tp, actual.Active)
+		}
+
+		if len(tp) == 0 && !tc.isPositive {
+			t.Errorf("active topics included unsupported topic(s) %v, list found: %v", tp, actual.Active)
+		}
+	}
+}
+
+func findMissingTopics(agianstList, sublist []string) []string {
+	missing := []string{}
+	for _, el := range sublist {
+		if !slices.Contains(agianstList, el) {
+			missing = append(missing, el)
+		}
+	}
+	return missing
+}

internal/api/features/types.go

@@ -0,0 +1,59 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2024 Red Hat, Inc.
+ */
+
+package features
+
+import (
+	"fmt"
+
+	goversion "github.com/aquasecurity/go-version/pkg/version"
+)
+
+const (
+	Version = "v4.17.0"
+)
+
+type Metadata struct {
+	// semantic versioning vX.Y.Z, e.g. v1.0.0
+	Version string `json:"version"`
+}
+
+type TopicInfo struct {
+	Metadata Metadata `json:"metadata"`
+	// unsorted list of topics active (supported)
+	Active []string `json:"active"`
+}
+
+func NewTopicInfo() TopicInfo {
+	return TopicInfo{
+		Metadata: Metadata{
+			Version: Version,
+		},
+	}
+}
+
+func (tp TopicInfo) Validate() error {
+	if tp.Metadata.Version == "" {
+		return fmt.Errorf("metadata: missing version")
+	}
+	if _, err := goversion.Parse(tp.Metadata.Version); err != nil {
+		return fmt.Errorf("metadata: malformed version: %w", err)
+	}
+	if len(tp.Active) == 0 {
+		return fmt.Errorf("missing active topics")
+	}
+	return nil
+}