Update KorMedMCQA v2.0

lm_eval/models/optimum_lm.py

@@ -71,9 +71,11 @@ def _create_model(
         else:
             model_kwargs["ov_config"] = {}
         model_kwargs["ov_config"].setdefault("CACHE_DIR", "")
-        if 'pipeline_parallel' in model_kwargs:
-            if model_kwargs['pipeline_parallel']:
-                model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = "PIPELINE_PARALLEL"
+        if "pipeline_parallel" in model_kwargs:
+            if model_kwargs["pipeline_parallel"]:
+                model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = (
+                    "PIPELINE_PARALLEL"
+                )
         model_file = Path(pretrained) / "openvino_model.xml"
         if model_file.exists():
             export = False

lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml

@@ -1,12 +1,12 @@
-group: kormedmcqa
-task:
-  - kormedmcqa_doctor
-  - kormedmcqa_nurse
-  - kormedmcqa_pharm
-  - kormedmcqa_dentist
-aggregate_metric_list:
-  - metric: exact_match
-    aggregation: mean
-    weight_by_size: true
-metadata:
-  version: 2.0
+group: kormedmcqa
+task:
+  - kormedmcqa_doctor
+  - kormedmcqa_nurse
+  - kormedmcqa_pharm
+  - kormedmcqa_dentist
+aggregate_metric_list:
+  - metric: exact_match
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 2.0

lm_eval/tasks/kormedmcqa/_template_yaml

@@ -1,33 +1,33 @@
-dataset_path : sean0042/KorMedMCQA
-test_split : test
-fewshot_split : fewshot
-fewshot_config:
-  sampler: first_n
-  doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n"
-  doc_to_target: ""
-output_type: generate_until
-doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： "
-doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}"
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-    regexes_to_ignore:
-      - " "
-      - "\n"
-generation_kwargs:
-  until:
-    - "Q:"
-    - "</s>"
-    - "<|im_end|>"
-    - "."
-    - "\n\n"
-  do_sample: false
-  temperature: 0.0
-  max_gen_toks: 1024
-metadata:
-  version: 2.0
-dataset_kwargs:
-  trust_remote_code: true
+dataset_path : sean0042/KorMedMCQA
+test_split : test
+fewshot_split : fewshot
+fewshot_config:
+  sampler: first_n
+  doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n"
+  doc_to_target: ""
+output_type: generate_until
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - " "
+      - "\n"
+generation_kwargs:
+  until:
+    - "Q:"
+    - "</s>"
+    - "<|im_end|>"
+    - "."
+    - "\n\n"
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 1024
+metadata:
+  version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/kormedmcqa/dentist.yaml

@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: dentist
-task: kormedmcqa_dentist
+include: _template_yaml
+dataset_name: dentist
+task: kormedmcqa_dentist

lm_eval/tasks/kormedmcqa/doctor.yaml

@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: doctor
-task: kormedmcqa_doctor
+include: _template_yaml
+dataset_name: doctor
+task: kormedmcqa_doctor

lm_eval/tasks/kormedmcqa/nurse.yaml

@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: nurse
-task: kormedmcqa_nurse
+include: _template_yaml
+dataset_name: nurse
+task: kormedmcqa_nurse

lm_eval/tasks/kormedmcqa/pharm.yaml

@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: pharm
-task: kormedmcqa_pharm
+include: _template_yaml
+dataset_name: pharm
+task: kormedmcqa_pharm