AraDICE task config file

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU.yaml

@@ -0,0 +1,12 @@
+group: AraDiCE_ArabicMMLU_egy
+task:
+- AraDiCE_ArabicMMLU_humanities_egy
+- AraDiCE_ArabicMMLU_language_egy
+- AraDiCE_ArabicMMLU_social-science_egy
+- AraDiCE_ArabicMMLU_stem_egy
+- AraDiCE_ArabicMMLU_other_egy
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_humanities_history.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_humanities_history"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_high_humanities_history_egy"
+"task_alias": "high humanities history"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_humanities_islamic-studies.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_humanities_islamic-studies"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_high_humanities_islamic-studies_egy"
+"task_alias": "high humanities islamic-studies"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_humanities_philosophy.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_humanities_philosophy"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_high_humanities_philosophy_egy"
+"task_alias": "high humanities philosophy"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_language_arabic-language.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_language_arabic-language"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_language_egy"
+"task": "AraDiCE_ArabicMMLU_high_language_arabic-language_egy"
+"task_alias": "high language arabic-language"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_social-science_civics.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_social-science_civics"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_high_social-science_civics_egy"
+"task_alias": "high social-science civics"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_social-science_economics.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_social-science_economics"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_high_social-science_economics_egy"
+"task_alias": "high social-science economics"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_social-science_geography.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_social-science_geography"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_high_social-science_geography_egy"
+"task_alias": "high social-science geography"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_stem_biology.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_stem_biology"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_high_stem_biology_egy"
+"task_alias": "high stem biology"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_stem_computer-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_stem_computer-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_high_stem_computer-science_egy"
+"task_alias": "high stem computer-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_stem_physics.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "high_stem_physics"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_high_stem_physics_egy"
+"task_alias": "high stem physics"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_humanities_history.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_humanities_history"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_middle_humanities_history_egy"
+"task_alias": "middle humanities history"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_humanities_islamic-studies.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_humanities_islamic-studies"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_middle_humanities_islamic-studies_egy"
+"task_alias": "middle humanities islamic-studies"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_language_arabic-language.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_language_arabic-language"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_language_egy"
+"task": "AraDiCE_ArabicMMLU_middle_language_arabic-language_egy"
+"task_alias": "middle language arabic-language"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_other_general-knowledge.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_other_general-knowledge"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_other_egy"
+"task": "AraDiCE_ArabicMMLU_middle_other_general-knowledge_egy"
+"task_alias": "middle other general-knowledge"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_social-science_civics.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_social-science_civics"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_middle_social-science_civics_egy"
+"task_alias": "middle social-science civics"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_social-science_economics.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_social-science_economics"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_middle_social-science_economics_egy"
+"task_alias": "middle social-science economics"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_social-science_geography.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_social-science_geography"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_middle_social-science_geography_egy"
+"task_alias": "middle social-science geography"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_social-science_social-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_social-science_social-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_middle_social-science_social-science_egy"
+"task_alias": "middle social-science social-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_stem_computer-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_stem_computer-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_middle_stem_computer-science_egy"
+"task_alias": "middle stem computer-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_middle_stem_natural-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "middle_stem_natural-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_middle_stem_natural-science_egy"
+"task_alias": "middle stem natural-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_na_humanities_islamic-studies.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "na_humanities_islamic-studies"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_na_humanities_islamic-studies_egy"
+"task_alias": "na humanities islamic-studies"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_na_language_arabic-language-general.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "na_language_arabic-language-general"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_language_egy"
+"task": "AraDiCE_ArabicMMLU_na_language_arabic-language-general_egy"
+"task_alias": "na language arabic-language-general"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_na_language_arabic-language-grammar.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "na_language_arabic-language-grammar"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_language_egy"
+"task": "AraDiCE_ArabicMMLU_na_language_arabic-language-grammar_egy"
+"task_alias": "na language arabic-language-grammar"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_na_other_driving-test.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "na_other_driving-test"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_other_egy"
+"task": "AraDiCE_ArabicMMLU_na_other_driving-test_egy"
+"task_alias": "na other driving-test"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_na_other_general-knowledge.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "na_other_general-knowledge"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_other_egy"
+"task": "AraDiCE_ArabicMMLU_na_other_general-knowledge_egy"
+"task_alias": "na other general-knowledge"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_humanities_history.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_humanities_history"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_primary_humanities_history_egy"
+"task_alias": "primary humanities history"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_humanities_islamic-studies.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_humanities_islamic-studies"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_primary_humanities_islamic-studies_egy"
+"task_alias": "primary humanities islamic-studies"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_language_arabic-language.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_language_arabic-language"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_language_egy"
+"task": "AraDiCE_ArabicMMLU_primary_language_arabic-language_egy"
+"task_alias": "primary language arabic-language"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_other_general-knowledge.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_other_general-knowledge"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_other_egy"
+"task": "AraDiCE_ArabicMMLU_primary_other_general-knowledge_egy"
+"task_alias": "primary other general-knowledge"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_social-science_geography.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_social-science_geography"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_primary_social-science_geography_egy"
+"task_alias": "primary social-science geography"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_social-science_social-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_social-science_social-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_social-science_egy"
+"task": "AraDiCE_ArabicMMLU_primary_social-science_social-science_egy"
+"task_alias": "primary social-science social-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_stem_computer-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_stem_computer-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_primary_stem_computer-science_egy"
+"task_alias": "primary stem computer-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_stem_math.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_stem_math"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_primary_stem_math_egy"
+"task_alias": "primary stem math"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_primary_stem_natural-science.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "primary_stem_natural-science"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_stem_egy"
+"task": "AraDiCE_ArabicMMLU_primary_stem_natural-science_egy"
+"task_alias": "primary stem natural-science"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_prof_humanities_law.yaml

@@ -0,0 +1,10 @@
+"dataset_name": "prof_humanities_law"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_prof_humanities_law_egy"
+"task_alias": "prof humanities law"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"