e2e: add a test for L2 cache cluster allocation

Signed-off-by: Antti Kervinen <[email protected]>
containers · Nov 8, 2024 · f9b7a24 · f9b7a24
1 parent d9f110a
commit f9b7a24
Show file tree

Hide file tree

Showing 3 changed files with 159 additions and 0 deletions.
diff --git a/...e/policies.test-suite/balloons/n4c16/test13-cacheclusters/balloons-2cpu-cacheclusters.cfg b/...e/policies.test-suite/balloons/n4c16/test13-cacheclusters/balloons-2cpu-cacheclusters.cfg
@@ -0,0 +1,39 @@
+config:
+  allocatorTopologyBalancing: true
+  reservedResources:
+    cpu: 750m
+  pinCPU: true
+  pinMemory: true
+  balloonTypes:
+  - name: l2burst
+    minCPUs: 1
+    maxCPUs: 4
+    shareIdleCPUsInSame: l2cache
+    preferSpreadingPods: true # no "same pod" affinity
+    preferNewBalloons: true # => every container to its own balloon instance
+  - name: l2pack
+    allocatorTopologyBalancing: false # create instances to the same L2 group
+    shareIdleCPUsInSame: l2cache
+    maxCPUs: 1
+  log:
+    debug:
+      - resource-manager
+      - cache
+      - policy
+      - sysfs
+    source: true
+    klog:
+      skip_headers: true
+extraEnv:
+  OVERRIDE_SYS_CACHES: '''[
+  {"cpusets": ["0-7", "8-15"], "level": 3, "size": "128M"},
+
+  {"cpusets": ["0-1", "2-3", "4-5", "6-7"], "level": 2, "size": "4M", "kind": "u"},
+  {"cpusets": ["8-9", "10-11", "12-13", "14-15"], "level": 2, "size": "4M", "kind": "u"},
+
+  {"cpusets": ["0-1", "2-3", "4-5", "6-7"], "level": 1, "size": "256k", "kind": "d"},
+  {"cpusets": ["0-1", "2-3", "4-5", "6-7"], "level": 1, "size": "128k", "kind": "i"},
+
+  {"cpusets": ["8-9", "10-11", "12-13", "14-15"], "level": 1, "size": "256k", "kind": "d"},
+  {"cpusets": ["8-9", "10-11", "12-13", "14-15"], "level": 1, "size": "128k", "kind": "i"}
+  ]'''
diff --git a/...e/policies.test-suite/balloons/n4c16/test13-cacheclusters/balloons-4cpu-cacheclusters.cfg b/...e/policies.test-suite/balloons/n4c16/test13-cacheclusters/balloons-4cpu-cacheclusters.cfg
@@ -0,0 +1,39 @@
+config:
+  allocatorTopologyBalancing: true
+  reservedResources:
+    cpu: 750m
+  pinCPU: true
+  pinMemory: true
+  balloonTypes:
+  - name: l2burst
+    minCPUs: 1
+    maxCPUs: 4
+    shareIdleCPUsInSame: l2cache
+    preferSpreadingPods: true # no "same pod" affinity
+    preferNewBalloons: true # => every container to its own balloon instance
+  - name: l2pack
+    allocatorTopologyBalancing: false # create instances to the same L2 group
+    shareIdleCPUsInSame: l2cache
+    maxCPUs: 1
+  log:
+    debug:
+      - resource-manager
+      - cache
+      - policy
+      - sysfs
+    source: true
+    klog:
+      skip_headers: true
+extraEnv:
+  OVERRIDE_SYS_CACHES: '''[
+  {"cpusets": ["0-7", "8-15"], "level": 3, "size": "128M"},
+
+  {"cpusets": ["0-3", "4-7"], "level": 2, "size": "8M"},
+  {"cpusets": ["8-11", "12-15"], "level": 2, "size": "8M"},
+
+  {"cpusets": ["0-1", "2-3", "4-5", "6-7"], "level": 1, "size": "256k", "kind": "d"},
+  {"cpusets": ["0-1", "2-3", "4-5", "6-7"], "level": 1, "size": "128k", "kind": "i"},
+
+  {"cpusets": ["8-9", "10-11", "12-13", "14-15"], "level": 1, "size": "256k", "kind": "d"},
+  {"cpusets": ["8-9", "10-11", "12-13", "14-15"], "level": 1, "size": "128k", "kind": "i"}
+  ]'''
diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh
@@ -0,0 +1,81 @@
+helm-terminate
+helm_config=$TEST_DIR/balloons-4cpu-cacheclusters.cfg helm-launch balloons
+
+cleanup() {
+    vm-command "kubectl delete pods --all --now"
+    helm-terminate
+    vm-command "rm -f /var/lib/nri-resource-policy/cache" || true
+}
+
+# pod0c{0,1,2}: one container per free L2 group
+CPUREQ="1500m" MEMREQ="100M" CPULIM="1500m" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2burst" CONTCOUNT=3 create balloons-busybox
+report allowed
+verify 'len(cpus["pod0c0"]) == 4' \
+       'len(cpus["pod0c0"]) == 4' \
+       'len(cpus["pod0c1"]) == 4' \
+       'disjoint_sets(nodes["pod0c0"], nodes["pod0c1"], nodes["pod0c2"])'
+
+# pod1c0: one more container to new L2 group, but this is shared with the reserved balloon of size 1.
+CPUREQ="2" MEMREQ="100M" CPULIM="2" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2burst" CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'len(cpus["pod1c0"]) == 3' \
+       'disjoint_sets(cpus["pod1c0"], cpus["pod0c0"], cpus["pod0c1"], cpus["pod0c2"])'
+
+# pod2c{0,1,2,3} fit two per l2pack balloon, all l2pack balloons should be in the same L2 cache.
+vm-command "kubectl delete pods --all --now"
+CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2pack" CONTCOUNT=4 create balloons-busybox
+report allowed
+verify 'len(cpus["pod2c0"]) == 2' \
+       'len(cpus["pod2c1"]) == 2' \
+       'len(cpus["pod2c2"]) == 2' \
+       'len(cpus["pod2c3"]) == 2' \
+       'len(set.intersection(cpus["pod2c0"], cpus["pod2c1"], cpus["pod2c2"], cpus["pod2c3"])) == 1' \
+       'len(set.union(cpus["pod2c0"], cpus["pod2c1"], cpus["pod2c2"], cpus["pod2c3"])) == 3' \
+       'nodes["pod2c0"] == nodes["pod2c1"] == nodes["pod2c2"] == nodes["pod2c3"]'
+
+# Due to packing, pod2c* should share the same L2 cache with the only existing balloon (the reserved balloon)
+# pod3c0: reserved-balloon container requires the reserved balloon to inflate from 1 to 2 CPUs.
+# Therefore there is no more shared idle CPU for pod2* containers.
+CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: reserved" CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'nodes["pod3c0"] == nodes["pod2c0"] == nodes["pod2c1"] == nodes["pod2c2"] == nodes["pod2c3"]' \
+       'len(cpus["pod3c0"]) == 2' \
+       'len(cpus["pod2c0"]) == 1' \
+       'len(cpus["pod2c1"]) == 1' \
+       'len(cpus["pod2c2"]) == 1' \
+       'len(cpus["pod2c3"]) == 1' \
+       'disjoint_sets(cpus["pod3c0"], cpus["pod2c0"])' \
+       'disjoint_sets(cpus["pod3c0"], cpus["pod2c1"])' \
+       'disjoint_sets(cpus["pod3c0"], cpus["pod2c2"])' \
+       'disjoint_sets(cpus["pod3c0"], cpus["pod2c3"])' \
+       'len(set.union(cpus["pod3c0"], cpus["pod2c0"], cpus["pod2c1"], cpus["pod2c2"], cpus["pod2c3"])) == 4'
+
+cleanup
+
+helm-terminate
+helm_config=$TEST_DIR/balloons-2cpu-cacheclusters.cfg helm-launch balloons
+
+# pod4c{0,1,2,3}: one container per free L2 group, this time L2 groups contain only single CPU cores
+CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2burst" CONTCOUNT=4 create balloons-busybox
+report allowed
+verify 'len(cpus["pod4c0"]) == 2' \
+       'len(cpus["pod4c1"]) == 2' \
+       'len(cpus["pod4c2"]) == 2' \
+       'len(cpus["pod4c3"]) == 2' \
+       'disjoint_sets(cpus["pod4c0"], cpus["pod4c1"], cpus["pod4c2"], cpus["pod4c3"])' \
+       'disjoint_sets(nodes["pod4c0"], nodes["pod4c1"], nodes["pod4c2"], nodes["pod4c3"])'
+# pod5c{0,1,2}: one container per free L2 group, fill three NUMA nodes
+CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2burst" CONTCOUNT=3 create balloons-busybox
+report allowed
+verify 'len(cpus["pod5c0"]) == 2' \
+       'len(cpus["pod5c1"]) == 2' \
+       'len(cpus["pod5c2"]) == 2' \
+       'disjoint_sets(cpus["pod5c0"], cpus["pod5c1"], cpus["pod5c2"], cpus["pod4c0"], cpus["pod4c1"], cpus["pod4c2"], cpus["pod4c3"])'
+
+cleanup