Add Aggregation for Kobest Benchmark

EleutherAI · Oct 31, 2024 · 48002c5 · 48002c5
1 parent 57272b6
commit 48002c5
Showing 1 changed file with 19 additions and 0 deletions.
diff --git a/lm_eval/tasks/kobest/_kobest.yaml b/lm_eval/tasks/kobest/_kobest.yaml
@@ -0,0 +1,19 @@
+group: kobest
+task:
+  - kobest_boolq
+  - kobest_copa
+  - kobest_hellaswag
+  - kobest_sentineg
+  - kobest_wic
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
+  - metric: f1
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0