test: add simple tests to verify isolcpus allocation

Signed-off-by: Feruzjon Muyassarov <[email protected]>

test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/balloons-isolcpus.cfg

@@ -0,0 +1,16 @@
+config:
+  balloonTypes:
+    - name: isolcpus
+      minCPUs: 1
+      minBalloons: 1
+      preferIsolCpus: true
+      namespaces:
+      - isolcpus
+  instrumentation:
+    httpEndpoint: :8891
+    prometheusExport: true
+  log:
+    debug:
+      - policy
+    klog:
+      skip_headers: true

test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh

@@ -0,0 +1,54 @@
+ns=isolcpus
+
+reboot-node() {
+    vm-reboot
+    timeout=120 host-wait-vm-ssh-server
+}
+
+restart-kubelet() {
+    vm-command "systemctl restart kubelet"
+    sleep 5
+    vm-wait-process --timeout 120 kube-apiserver
+    vm-command "cilium status --wait --wait-duration=120s --interactive=false"
+}
+
+vm-command "grep isolcpus=0,1 /proc/cmdline" || {
+    if [[ "$distro" == *fedora* ]]; then
+        fedora-set-kernel-cmdline "isolcpus=0,1"
+    else
+        ubuntu-set-kernel-cmdline "isolcpus=0,1"
+    fi
+    vm-reboot
+    vm-command "grep isolcpus=0,1 /proc/cmdline" || {
+        error "failed to set isolcpus kernel commandline parameter"
+    }
+    restart-kubelet
+}
+
+helm-terminate
+helm_config=${TEST_DIR}/balloons-isolcpus.cfg helm-launch balloons
+vm-command "kubectl create namespace $ns"
+
+# pod0: runs on system isolated CPUs
+CONTCOUNT=2 namespace="$ns" create balloons-busybox
+report allowed
+verify "cpus['pod0c0'] == {'cpu00', 'cpu01'}"
+
+# pod1: should run on non-isolated CPUs
+CONTCOUNT=1 namespace="$ns" create balloons-busybox
+report allowed
+verify "cpus['pod1c0'] != {'cpu00', 'cpu01'}"
+
+cleanup() {
+    vm-command "kubectl delete namespace $ns"
+    if [[ "$distro" == *fedora* ]]; then
+        fedora-set-kernel-cmdline ""
+    else
+        ubuntu-set-kernel-cmdline ""
+    fi
+    reboot-node
+    vm-command "grep isolcpus= /proc/cmdline" || {
+        error "failed to unset isolcpus kernel commandline parameter"
+    }
+}
+cleanup

test/e2e/run.sh

@@ -915,6 +915,85 @@ $py_assertion
     return 0
 }
 
+fedora-set-kernel-cmdline() {
+    local e2e_defaults="$*"
+    vm-command "mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub"
+    vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\" # by e2e:fedora-set-kernel-cmdline' >> /etc/default/grub" || {
+        command-error "writing new command line parameters failed"
+    }
+    vm-command "grub2-mkconfig -o /boot/grub2/grub.cfg" || {
+        command-error "updating grub failed"
+    }
+}
+
+host-wait-vm-ssh-server() {
+    local _vagrantdir="$1"
+    local _timeout=${timeout:-10}
+    local _waited
+
+    if [ -z "$_vagrantdir" ]; then
+        echo 1>&2 "host-wait-vm-ssh-server: missing vagrant directory"
+        return 1
+    fi
+
+    _waited=0; while (( $_waited < $_timeout )); do
+        if [ ! -f $_vagrantdir/.ssh-config ]; then
+            sleep 1
+        else
+            ssh -o ConnectTimeout=1 -F $_vagrantdir/.ssh-config node true
+            if [ $? == 0 ]; then
+                return 0
+            fi
+        fi
+        _waited=$((_waited + 1))
+    done
+
+    echo 1>&2 "host-wait-vm-ssh-server: timeout waiting for $_vagrantdir ssh server"
+    return 1
+}
+
+# fedora-set-kernel-cmdline() {
+#     local e2e_defaults="$1"
+#     vm-command "mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub"
+#     vm-command "sed -i '/${e2e_defaults}/d' /etc/default/grub"
+#     vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\"' >> /etc/default/grub" || {
+#         command-error "writing new command line parameters failed"
+#     }
+#     vm-command "grub2-mkconfig -o /boot/grub2/grub.cfg" || {
+#         command-error "updating grub failed"
+#     }
+# }
+
+fedora-set-kernel-cmdline() {
+    local e2e_defaults="$*"
+    vm-command "mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub"
+    vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\" # by e2e:fedora-set-kernel-cmdline' >> /etc/default/grub" || {
+        command-error "writing new command line parameters failed"
+    }
+    vm-command "grub2-mkconfig -o /boot/grub2/grub.cfg" || {
+        command-error "updating grub failed"
+    }
+}
+
+ubuntu-set-kernel-cmdline() {
+    local e2e_defaults="$*"
+    vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\"' > /etc/default/grub.d/60-e2e-defaults.cfg" || {
+        command-error "writing new command line parameters failed"
+    }
+    vm-command "update-grub" || {
+        command-error "updating grub failed"
+    }
+}
+
+vm-reboot() { # script API
+    # Usage: vm-reboot
+    #
+    # Reboots the virtual machine and waits that the ssh server starts
+    # responding again.
+    vm-command "reboot"
+    sleep 5
+}
+
 # Defaults to use in case the test case does not define these values.
 yaml_in_defaults="CPU=1 MEM=100M ISO=true CPUREQ=1 CPULIM=2 MEMREQ=100M MEMLIM=200M CONTCOUNT=1"