From 5280bc7b2913a5ccf9c4c9d8b1e1d51eab4ff24b Mon Sep 17 00:00:00 2001
From: Sai Sindhur Malleni <smalleni@redhat.com>
Date: Thu, 18 May 2023 02:20:07 +0000
Subject: [PATCH] Disable validation-webhook daemonset for managed clusters

When running workloads on managed clusters with cluster-admin user, we are seeing an error like "admission webhook
"regular-user-validation.managed.openshift.io" denied the request" when trying add a label to a
node directly by using oc label node. The recommended way to add labels
to nodes on managed ROSA clusters is by editing the machinepool.
However, the Default machinepool cannot be edited to add labels and we
see an error like "Labels cannot be updated on the Default machine pool".
The only way to add a label is by disabling the
validation-webhook daemonset and thereby admission control in the
openshift-validation-webhook project that only exists on managed services
clusters. We disable the daemonset by adding a fake nodeSelector before
labeling the nodes and remove the nodeSelector after unlabeling the nodes.
Adding a nodeSelector on top of the existing nodeAffinity means that both
the conditions needs to be met for a pod to be scheduled. Also by adding
the nodeSelector, the spec is not overwritten during reconcillation whereas
changes to nodeAffinity are being overwritten.

This change4 is important for managed clusters as we don't always have access to kubeconfig (when
running in prow for example).

Signed-off-by: Sai Sindhur Malleni <smalleni@redhat.com>
---
 workloads/kube-burner-ocp-wrapper/run.sh | 31 +++++++++++++++++++++++-
 workloads/kube-burner/common.sh          | 19 +++++++++++++++
 workloads/kube-burner/run.sh             | 10 ++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/workloads/kube-burner-ocp-wrapper/run.sh b/workloads/kube-burner-ocp-wrapper/run.sh
index 0a713546..ce9de487 100755
--- a/workloads/kube-burner-ocp-wrapper/run.sh
+++ b/workloads/kube-burner-ocp-wrapper/run.sh
@@ -14,11 +14,32 @@ EXTRA_FLAGS=${EXTRA_FLAGS:-}
 UUID=$(uuidgen)
 KUBE_DIR=${KUBE_DIR:-/tmp}
 
+check_managed_cluster() {
+   status=$(oc get infrastructure/cluster -o=jsonpath='{.status.platformStatus.*.resourceTags[0]}')
+   if [[ $status =~ managed ]]; then
+      echo "Detected a Managed Cluster"
+      managed=true
+   fi
+}
+
+remove_managed_webhook_validation() {
+   echo "Disabling validation-webhook for Managed cluster"
+   oc patch -n openshift-validation-webhook daemonset validation-webhook -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
+   
+}
+
+add_managed_webhook_validation() {
+    echo "Enabling validation-webhook for Managed cluster"
+    oc patch -n openshift-validation-webhook daemonset validation-webhook --type json -p '[{ "op": "remove", "path": "/spec/template/spec/nodeSelector" }]'
+}
+
 download_binary(){
   KUBE_BURNER_URL=https://github.com/cloud-bulldozer/kube-burner/releases/download/v${KUBE_BURNER_VERSION}/kube-burner-${KUBE_BURNER_VERSION}-Linux-x86_64.tar.gz
   curl -sS -L ${KUBE_BURNER_URL} | tar -xzC ${KUBE_DIR}/ kube-burner
 }
 
+check_managed_cluster
+
 hypershift(){
   echo "HyperShift detected"
   # Get hosted cluster ID and name
@@ -96,5 +117,13 @@ EOF
 )
 curl -k -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/ripsaw-kube-burner/_doc -d "${METADATA}" -o /dev/null
 
+if [[ $managed == true ]]; then
+   remove_managed_webhook_validation
+fi
+
 echo $cmd
-exec $cmd
+$cmd
+
+if [[ $managed == true ]]; then
+   add_managed_webhook_validation
+fi
diff --git a/workloads/kube-burner/common.sh b/workloads/kube-burner/common.sh
index 01338930..aaab3080 100644
--- a/workloads/kube-burner/common.sh
+++ b/workloads/kube-burner/common.sh
@@ -94,6 +94,25 @@ run_workload() {
   gen_metadata ${WORKLOAD} ${start_date} $(date +%s%3N)
 }
 
+check_managed_cluster() {
+   status=$(oc get infrastructure/cluster -o=jsonpath='{.status.platformStatus.*.resourceTags[0]}')
+   if [[ $status =~ managed ]]; then
+      log "Detected a Managed Cluster"
+      managed=true
+   fi
+}
+
+remove_managed_webhook_validation() {
+   log "Disabling validation-webhook for Managed cluster"
+   oc patch -n openshift-validation-webhook daemonset validation-webhook -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
+   
+}
+
+add_managed_webhook_validation() {
+    log "Enabling validation-webhook for Managed cluster"
+    oc patch -n openshift-validation-webhook daemonset validation-webhook --type json -p '[{ "op": "remove", "path": "/spec/template/spec/nodeSelector" }]'
+}
+
 find_running_pods_num() {
   pod_count=0
   # The next statement outputs something similar to:
diff --git a/workloads/kube-burner/run.sh b/workloads/kube-burner/run.sh
index cc05c69b..076bf137 100755
--- a/workloads/kube-burner/run.sh
+++ b/workloads/kube-burner/run.sh
@@ -5,6 +5,12 @@
 . ../../utils/compare.sh
 
 label=""
+
+check_managed_cluster
+if [[ $managed == true ]]; then
+   remove_managed_webhook_validation
+fi
+
 case ${WORKLOAD} in
   cluster-density)
     WORKLOAD_TEMPLATE=workloads/cluster-density/cluster-density.yml
@@ -163,6 +169,10 @@ else
   run_workload
 fi
 
+if [[ $managed == true ]]; then
+   add_managed_webhook_validation
+fi
+
 if [[ ${CLEANUP_WHEN_FINISH} == "true" ]]; then
   cleanup
   if [[ ${WORKLOAD} == node-density* || ${WORKLOAD} == pod-density-heavy ]]; then