From 7b76af743d8d654a0888747d2e1a6a82ec1aaefc Mon Sep 17 00:00:00 2001
From: GyoukChu <kyouwook@kaist.ac.kr>
Date: Fri, 13 Dec 2024 00:40:28 +0900
Subject: [PATCH] Update KorMedMCQA v2.0

---
 lm_eval/models/optimum_lm.py              |  8 +--
 lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml | 24 ++++-----
 lm_eval/tasks/kormedmcqa/_template_yaml   | 66 +++++++++++------------
 lm_eval/tasks/kormedmcqa/dentist.yaml     |  6 +--
 lm_eval/tasks/kormedmcqa/doctor.yaml      |  6 +--
 lm_eval/tasks/kormedmcqa/nurse.yaml       |  6 +--
 lm_eval/tasks/kormedmcqa/pharm.yaml       |  6 +--
 7 files changed, 62 insertions(+), 60 deletions(-)

diff --git a/lm_eval/models/optimum_lm.py b/lm_eval/models/optimum_lm.py
index 7a733aa865..b13b321f25 100644
--- a/lm_eval/models/optimum_lm.py
+++ b/lm_eval/models/optimum_lm.py
@@ -71,9 +71,11 @@ def _create_model(
         else:
             model_kwargs["ov_config"] = {}
         model_kwargs["ov_config"].setdefault("CACHE_DIR", "")
-        if 'pipeline_parallel' in model_kwargs:
-            if model_kwargs['pipeline_parallel']:
-                model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = "PIPELINE_PARALLEL"
+        if "pipeline_parallel" in model_kwargs:
+            if model_kwargs["pipeline_parallel"]:
+                model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = (
+                    "PIPELINE_PARALLEL"
+                )
         model_file = Path(pretrained) / "openvino_model.xml"
         if model_file.exists():
             export = False
diff --git a/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml
index 08c150cf35..cac2329e75 100644
--- a/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml
+++ b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml
@@ -1,12 +1,12 @@
-group: kormedmcqa
-task:
-  - kormedmcqa_doctor
-  - kormedmcqa_nurse
-  - kormedmcqa_pharm
-  - kormedmcqa_dentist
-aggregate_metric_list:
-  - metric: exact_match
-    aggregation: mean
-    weight_by_size: true
-metadata:
-  version: 2.0
\ No newline at end of file
+group: kormedmcqa
+task:
+  - kormedmcqa_doctor
+  - kormedmcqa_nurse
+  - kormedmcqa_pharm
+  - kormedmcqa_dentist
+aggregate_metric_list:
+  - metric: exact_match
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 2.0
diff --git a/lm_eval/tasks/kormedmcqa/_template_yaml b/lm_eval/tasks/kormedmcqa/_template_yaml
index 4aa6ab5bd3..1dae2062d1 100644
--- a/lm_eval/tasks/kormedmcqa/_template_yaml
+++ b/lm_eval/tasks/kormedmcqa/_template_yaml
@@ -1,33 +1,33 @@
-dataset_path : sean0042/KorMedMCQA
-test_split : test
-fewshot_split : fewshot
-fewshot_config:
-  sampler: first_n
-  doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n"
-  doc_to_target: ""
-output_type: generate_until
-doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： "
-doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}"
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-    regexes_to_ignore:
-      - " "
-      - "\n"
-generation_kwargs:
-  until:
-    - "Q:"
-    - "</s>"
-    - "<|im_end|>"
-    - "."
-    - "\n\n"
-  do_sample: false
-  temperature: 0.0
-  max_gen_toks: 1024
-metadata:
-  version: 2.0
-dataset_kwargs:
-  trust_remote_code: true
\ No newline at end of file
+dataset_path : sean0042/KorMedMCQA
+test_split : test
+fewshot_split : fewshot
+fewshot_config:
+  sampler: first_n
+  doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답： {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n"
+  doc_to_target: ""
+output_type: generate_until
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - " "
+      - "\n"
+generation_kwargs:
+  until:
+    - "Q:"
+    - "</s>"
+    - "<|im_end|>"
+    - "."
+    - "\n\n"
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 1024
+metadata:
+  version: 2.0
+dataset_kwargs:
+  trust_remote_code: true
diff --git a/lm_eval/tasks/kormedmcqa/dentist.yaml b/lm_eval/tasks/kormedmcqa/dentist.yaml
index 777b1ab722..6a46c7715a 100644
--- a/lm_eval/tasks/kormedmcqa/dentist.yaml
+++ b/lm_eval/tasks/kormedmcqa/dentist.yaml
@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: dentist
-task: kormedmcqa_dentist
\ No newline at end of file
+include: _template_yaml
+dataset_name: dentist
+task: kormedmcqa_dentist
diff --git a/lm_eval/tasks/kormedmcqa/doctor.yaml b/lm_eval/tasks/kormedmcqa/doctor.yaml
index 8d5340bf17..aac30e4cdf 100644
--- a/lm_eval/tasks/kormedmcqa/doctor.yaml
+++ b/lm_eval/tasks/kormedmcqa/doctor.yaml
@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: doctor
-task: kormedmcqa_doctor
\ No newline at end of file
+include: _template_yaml
+dataset_name: doctor
+task: kormedmcqa_doctor
diff --git a/lm_eval/tasks/kormedmcqa/nurse.yaml b/lm_eval/tasks/kormedmcqa/nurse.yaml
index 035245b3dc..95894a5d73 100644
--- a/lm_eval/tasks/kormedmcqa/nurse.yaml
+++ b/lm_eval/tasks/kormedmcqa/nurse.yaml
@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: nurse
-task: kormedmcqa_nurse
\ No newline at end of file
+include: _template_yaml
+dataset_name: nurse
+task: kormedmcqa_nurse
diff --git a/lm_eval/tasks/kormedmcqa/pharm.yaml b/lm_eval/tasks/kormedmcqa/pharm.yaml
index 6d97d5bdb9..8075fae3f8 100644
--- a/lm_eval/tasks/kormedmcqa/pharm.yaml
+++ b/lm_eval/tasks/kormedmcqa/pharm.yaml
@@ -1,3 +1,3 @@
-include: _template_yaml
-dataset_name: pharm
-task: kormedmcqa_pharm
\ No newline at end of file
+include: _template_yaml
+dataset_name: pharm
+task: kormedmcqa_pharm