diff --git a/lm_eval/models/optimum_lm.py b/lm_eval/models/optimum_lm.py index 7a733aa865..b13b321f25 100644 --- a/lm_eval/models/optimum_lm.py +++ b/lm_eval/models/optimum_lm.py @@ -71,9 +71,11 @@ def _create_model( else: model_kwargs["ov_config"] = {} model_kwargs["ov_config"].setdefault("CACHE_DIR", "") - if 'pipeline_parallel' in model_kwargs: - if model_kwargs['pipeline_parallel']: - model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = "PIPELINE_PARALLEL" + if "pipeline_parallel" in model_kwargs: + if model_kwargs["pipeline_parallel"]: + model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = ( + "PIPELINE_PARALLEL" + ) model_file = Path(pretrained) / "openvino_model.xml" if model_file.exists(): export = False diff --git a/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml index 08c150cf35..cac2329e75 100644 --- a/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml +++ b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml @@ -1,12 +1,12 @@ -group: kormedmcqa -task: - - kormedmcqa_doctor - - kormedmcqa_nurse - - kormedmcqa_pharm - - kormedmcqa_dentist -aggregate_metric_list: - - metric: exact_match - aggregation: mean - weight_by_size: true -metadata: - version: 2.0 \ No newline at end of file +group: kormedmcqa +task: + - kormedmcqa_doctor + - kormedmcqa_nurse + - kormedmcqa_pharm + - kormedmcqa_dentist +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/kormedmcqa/_template_yaml b/lm_eval/tasks/kormedmcqa/_template_yaml index 4aa6ab5bd3..1dae2062d1 100644 --- a/lm_eval/tasks/kormedmcqa/_template_yaml +++ b/lm_eval/tasks/kormedmcqa/_template_yaml @@ -1,33 +1,33 @@ -dataset_path : sean0042/KorMedMCQA -test_split : test -fewshot_split : fewshot -fewshot_config: - sampler: first_n - doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답: {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n" - doc_to_target: "" -output_type: generate_until -doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답: " -doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}" -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - regexes_to_ignore: - - " " - - "\n" -generation_kwargs: - until: - - "Q:" - - "" - - "<|im_end|>" - - "." - - "\n\n" - do_sample: false - temperature: 0.0 - max_gen_toks: 1024 -metadata: - version: 2.0 -dataset_kwargs: - trust_remote_code: true \ No newline at end of file +dataset_path : sean0042/KorMedMCQA +test_split : test +fewshot_split : fewshot +fewshot_config: + sampler: first_n + doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답: {{['A', 'B', 'C', 'D', 'E'][answer-1]}}\n\n" + doc_to_target: "" +output_type: generate_until +doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답:" +doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - " " + - "\n" +generation_kwargs: + until: + - "Q:" + - "" + - "<|im_end|>" + - "." + - "\n\n" + do_sample: false + temperature: 0.0 + max_gen_toks: 1024 +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/kormedmcqa/dentist.yaml b/lm_eval/tasks/kormedmcqa/dentist.yaml index 777b1ab722..6a46c7715a 100644 --- a/lm_eval/tasks/kormedmcqa/dentist.yaml +++ b/lm_eval/tasks/kormedmcqa/dentist.yaml @@ -1,3 +1,3 @@ -include: _template_yaml -dataset_name: dentist -task: kormedmcqa_dentist \ No newline at end of file +include: _template_yaml +dataset_name: dentist +task: kormedmcqa_dentist diff --git a/lm_eval/tasks/kormedmcqa/doctor.yaml b/lm_eval/tasks/kormedmcqa/doctor.yaml index 8d5340bf17..aac30e4cdf 100644 --- a/lm_eval/tasks/kormedmcqa/doctor.yaml +++ b/lm_eval/tasks/kormedmcqa/doctor.yaml @@ -1,3 +1,3 @@ -include: _template_yaml -dataset_name: doctor -task: kormedmcqa_doctor \ No newline at end of file +include: _template_yaml +dataset_name: doctor +task: kormedmcqa_doctor diff --git a/lm_eval/tasks/kormedmcqa/nurse.yaml b/lm_eval/tasks/kormedmcqa/nurse.yaml index 035245b3dc..95894a5d73 100644 --- a/lm_eval/tasks/kormedmcqa/nurse.yaml +++ b/lm_eval/tasks/kormedmcqa/nurse.yaml @@ -1,3 +1,3 @@ -include: _template_yaml -dataset_name: nurse -task: kormedmcqa_nurse \ No newline at end of file +include: _template_yaml +dataset_name: nurse +task: kormedmcqa_nurse diff --git a/lm_eval/tasks/kormedmcqa/pharm.yaml b/lm_eval/tasks/kormedmcqa/pharm.yaml index 6d97d5bdb9..8075fae3f8 100644 --- a/lm_eval/tasks/kormedmcqa/pharm.yaml +++ b/lm_eval/tasks/kormedmcqa/pharm.yaml @@ -1,3 +1,3 @@ -include: _template_yaml -dataset_name: pharm -task: kormedmcqa_pharm \ No newline at end of file +include: _template_yaml +dataset_name: pharm +task: kormedmcqa_pharm