updated file version | added args to exact_match

EleutherAI · Dec 4, 2024 · 578dbdb · 578dbdb
1 parent 97dccd3
commit 578dbdb
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 7 deletions.
diff --git a/lm_eval/tasks/mmlu/generative/_default_template_yaml b/lm_eval/tasks/mmlu/generative/_default_template_yaml
@@ -14,6 +14,8 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+    ignore_punctuation: true
+    ignore_case: true
 filter_list:
   - name: get_response
     filter:
@@ -27,6 +29,6 @@ filter_list:
         regex_pattern: "^(.*?)\\s*$"
       - function: take_first
 metadata:
-  version: 2.0
+  version: 3.0
 dataset_kwargs:
   trust_remote_code: true
diff --git a/lm_eval/tasks/mmlu/generative/_mmlu.yaml b/lm_eval/tasks/mmlu/generative/_mmlu.yaml
@@ -6,33 +6,43 @@ task:
       - mmlu_stem_generative
     aggregate_metric_list:
       - metric: exact_match
-        weight_by_size: True
+        weight_by_size: true
+        ignore_punctuation: true
+        ignore_case: true
         filter_list: get_response
   - group: other
     task:
       - mmlu_other_generative
     aggregate_metric_list:
       - metric: exact_match
-        weight_by_size: True
+        weight_by_size: true
+        ignore_punctuation: true
+        ignore_case: true
         filter_list: get_response
   - group: social sciences
     task:
       - mmlu_social_sciences_generative
     aggregate_metric_list:
       - metric: exact_match
-        weight_by_size: True
+        weight_by_size: true
+        ignore_punctuation: true
+        ignore_case: true
         filter_list: get_response
   - group: humanities
     task:
       - mmlu_humanities_generative
     aggregate_metric_list:
       - metric: exact_match
-        weight_by_size: True
+        weight_by_size: true
+        ignore_punctuation: true
+        ignore_case: true
         filter_list: get_response
 aggregate_metric_list:
   - aggregation: mean
     metric: exact_match
-    weight_by_size: True
+    weight_by_size: true
+    ignore_punctuation: true
+    ignore_case: true
     filter_list: get_response
 metadata:
-  version: 2
+  version: 3