From da8517b5c0f6d7cc930e9b2930dc0b48ac42cd9d Mon Sep 17 00:00:00 2001 From: tryumanshow Date: Thu, 31 Oct 2024 13:29:26 +0900 Subject: [PATCH] Add Aggregation for Kobest Benchmark --- lm_eval/tasks/kobest/_kobest.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lm_eval/tasks/kobest/_kobest.yaml diff --git a/lm_eval/tasks/kobest/_kobest.yaml b/lm_eval/tasks/kobest/_kobest.yaml new file mode 100644 index 0000000000..cf23f6643a --- /dev/null +++ b/lm_eval/tasks/kobest/_kobest.yaml @@ -0,0 +1,19 @@ +group: kobest +task: + - kobest_boolq + - kobest_copa + - kobest_hellaswag + - kobest_sentineg + - kobest_wic +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true + - metric: f1 + aggregation: mean + weight_by_size: true +metadata: + version: 1.0