{ "model_name": "kawk500m-instruct-v1", "model_dir": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer_dir": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "elapsed_seconds": 907.9745282999938, "evaluation": { "results": { "kobest_boolq": { "name": "kobest_boolq", "alias": "kobest_boolq", "sample_len": 1404, "acc,none": 0.5135327635327636, "acc_stderr,none": 0.013343877331261893, "f1,none": 0.4553178437236408, "f1_stderr,none": "N/A" }, "kobest_copa": { "name": "kobest_copa", "alias": "kobest_copa", "sample_len": 1000, "acc,none": 0.635, "acc_stderr,none": 0.015231776226264848, "f1,none": 0.6335455422572189, "f1_stderr,none": "N/A" }, "kobest_hellaswag": { "name": "kobest_hellaswag", "alias": "kobest_hellaswag", "sample_len": 500, "acc,none": 0.354, "acc_stderr,none": 0.02140758204791649, "f1,none": 0.3513485918054787, "f1_stderr,none": "N/A", "acc_norm,none": 0.474, "acc_norm_stderr,none": 0.022352791650914174 }, "kobest_sentineg": { "name": "kobest_sentineg", "alias": "kobest_sentineg", "sample_len": 397, "acc,none": 0.6045340050377834, "acc_stderr,none": 0.024570689701367988, "f1,none": 0.592580971990718, "f1_stderr,none": "N/A" }, "kobest_wic": { "name": "kobest_wic", "alias": "kobest_wic", "sample_len": 1260, "acc,none": 0.4865079365079365, "acc_stderr,none": 0.014086365971849238, "f1,none": 0.3561228043986665, "f1_stderr,none": "N/A" }, "kmmlu_biology": { "name": "kmmlu_biology", "alias": "kmmlu_biology", "sample_len": 1000, "acc,none": 0.204, "acc_stderr,none": 0.012749374359024311 }, "kmmlu_chemical_engineering": { "name": "kmmlu_chemical_engineering", "alias": "kmmlu_chemical_engineering", "sample_len": 1000, "acc,none": 0.215, "acc_stderr,none": 0.01299784381903183 }, "kmmlu_chemistry": { "name": "kmmlu_chemistry", "alias": "kmmlu_chemistry", "sample_len": 600, "acc,none": 0.21666666666666667, "acc_stderr,none": 0.016832783728500004 }, "kmmlu_civil_engineering": { "name": "kmmlu_civil_engineering", "alias": "kmmlu_civil_engineering", "sample_len": 1000, "acc,none": 0.084, "acc_stderr,none": 0.008776162089491113 }, "kmmlu_computer_science": { "name": "kmmlu_computer_science", "alias": "kmmlu_computer_science", "sample_len": 1000, "acc,none": 0.086, "acc_stderr,none": 0.008870325962594761 }, "kmmlu_ecology": { "name": "kmmlu_ecology", "alias": "kmmlu_ecology", "sample_len": 1000, "acc,none": 0.122, "acc_stderr,none": 0.010354864712936776 }, "kmmlu_electrical_engineering": { "name": "kmmlu_electrical_engineering", "alias": "kmmlu_electrical_engineering", "sample_len": 1000, "acc,none": 0.082, "acc_stderr,none": 0.008680515615523732 }, "kmmlu_information_technology": { "name": "kmmlu_information_technology", "alias": "kmmlu_information_technology", "sample_len": 1000, "acc,none": 0.104, "acc_stderr,none": 0.009658016218524242 }, "kmmlu_materials_engineering": { "name": "kmmlu_materials_engineering", "alias": "kmmlu_materials_engineering", "sample_len": 1000, "acc,none": 0.157, "acc_stderr,none": 0.011510146979230262 }, "kmmlu_math": { "name": "kmmlu_math", "alias": "kmmlu_math", "sample_len": 300, "acc,none": 0.26666666666666666, "acc_stderr,none": 0.02557404853322572 }, "kmmlu_mechanical_engineering": { "name": "kmmlu_mechanical_engineering", "alias": "kmmlu_mechanical_engineering", "sample_len": 1000, "acc,none": 0.102, "acc_stderr,none": 0.009575368801653944 }, "kmmlu_agricultural_sciences": { "name": "kmmlu_agricultural_sciences", "alias": "kmmlu_agricultural_sciences", "sample_len": 1000, "acc,none": 0.174, "acc_stderr,none": 0.011994493230973449 }, "kmmlu_construction": { "name": "kmmlu_construction", "alias": "kmmlu_construction", "sample_len": 1000, "acc,none": 0.111, "acc_stderr,none": 0.009938701010583716 }, "kmmlu_fashion": { "name": "kmmlu_fashion", "alias": "kmmlu_fashion", "sample_len": 1000, "acc,none": 0.186, "acc_stderr,none": 0.012310790208412926 }, "kmmlu_food_processing": { "name": "kmmlu_food_processing", "alias": "kmmlu_food_processing", "sample_len": 1000, "acc,none": 0.176, "acc_stderr,none": 0.012048616898597498 }, "kmmlu_health": { "name": "kmmlu_health", "alias": "kmmlu_health", "sample_len": 100, "acc,none": 0.25, "acc_stderr,none": 0.04351941398892446 }, "kmmlu_interior_architecture_and_design": { "name": "kmmlu_interior_architecture_and_design", "alias": "kmmlu_interior_architecture_and_design", "sample_len": 1000, "acc,none": 0.111, "acc_stderr,none": 0.009938701010583716 }, "kmmlu_marketing": { "name": "kmmlu_marketing", "alias": "kmmlu_marketing", "sample_len": 1000, "acc,none": 0.131, "acc_stderr,none": 0.0106748748448379 }, "kmmlu_patent": { "name": "kmmlu_patent", "alias": "kmmlu_patent", "sample_len": 100, "acc,none": 0.25, "acc_stderr,none": 0.04351941398892446 }, "kmmlu_public_safety": { "name": "kmmlu_public_safety", "alias": "kmmlu_public_safety", "sample_len": 1000, "acc,none": 0.116, "acc_stderr,none": 0.010131468138756924 }, "kmmlu_real_estate": { "name": "kmmlu_real_estate", "alias": "kmmlu_real_estate", "sample_len": 200, "acc,none": 0.22, "acc_stderr,none": 0.02936514188266327 }, "kmmlu_refrigerating_machinery": { "name": "kmmlu_refrigerating_machinery", "alias": "kmmlu_refrigerating_machinery", "sample_len": 1000, "acc,none": 0.157, "acc_stderr,none": 0.011510146979230262 }, "kmmlu_aviation_engineering_and_maintenance": { "name": "kmmlu_aviation_engineering_and_maintenance", "alias": "kmmlu_aviation_engineering_and_maintenance", "sample_len": 1000, "acc,none": 0.144, "acc_stderr,none": 0.01110798754893916 }, "kmmlu_electronics_engineering": { "name": "kmmlu_electronics_engineering", "alias": "kmmlu_electronics_engineering", "sample_len": 1000, "acc,none": 0.092, "acc_stderr,none": 0.009144376393151129 }, "kmmlu_energy_management": { "name": "kmmlu_energy_management", "alias": "kmmlu_energy_management", "sample_len": 1000, "acc,none": 0.207, "acc_stderr,none": 0.012818553557844009 }, "kmmlu_environmental_science": { "name": "kmmlu_environmental_science", "alias": "kmmlu_environmental_science", "sample_len": 1000, "acc,none": 0.101, "acc_stderr,none": 0.009533618929341046 }, "kmmlu_gas_technology_and_engineering": { "name": "kmmlu_gas_technology_and_engineering", "alias": "kmmlu_gas_technology_and_engineering", "sample_len": 1000, "acc,none": 0.118, "acc_stderr,none": 0.010206869264381718 }, "kmmlu_geomatics": { "name": "kmmlu_geomatics", "alias": "kmmlu_geomatics", "sample_len": 1000, "acc,none": 0.125, "acc_stderr,none": 0.010463483381956722 }, "kmmlu_industrial_engineer": { "name": "kmmlu_industrial_engineer", "alias": "kmmlu_industrial_engineer", "sample_len": 1000, "acc,none": 0.071, "acc_stderr,none": 0.008125578442487959 }, "kmmlu_machine_design_and_manufacturing": { "name": "kmmlu_machine_design_and_manufacturing", "alias": "kmmlu_machine_design_and_manufacturing", "sample_len": 1000, "acc,none": 0.123, "acc_stderr,none": 0.010391293421849803 }, "kmmlu_maritime_engineering": { "name": "kmmlu_maritime_engineering", "alias": "kmmlu_maritime_engineering", "sample_len": 600, "acc,none": 0.15666666666666668, "acc_stderr,none": 0.014851643766757317 }, "kmmlu_nondestructive_testing": { "name": "kmmlu_nondestructive_testing", "alias": "kmmlu_nondestructive_testing", "sample_len": 1000, "acc,none": 0.15, "acc_stderr,none": 0.011297239823409416 }, "kmmlu_railway_and_automotive_engineering": { "name": "kmmlu_railway_and_automotive_engineering", "alias": "kmmlu_railway_and_automotive_engineering", "sample_len": 1000, "acc,none": 0.136, "acc_stderr,none": 0.01084535023047304 }, "kmmlu_telecommunications_and_wireless_technology": { "name": "kmmlu_telecommunications_and_wireless_technology", "alias": "kmmlu_telecommunications_and_wireless_technology", "sample_len": 1000, "acc,none": 0.105, "acc_stderr,none": 0.00969892102602496 }, "kmmlu_accounting": { "name": "kmmlu_accounting", "alias": "kmmlu_accounting", "sample_len": 100, "acc,none": 0.24, "acc_stderr,none": 0.04292346959909278 }, "kmmlu_criminal_law": { "name": "kmmlu_criminal_law", "alias": "kmmlu_criminal_law", "sample_len": 200, "acc,none": 0.195, "acc_stderr,none": 0.028085923439997246 }, "kmmlu_economics": { "name": "kmmlu_economics", "alias": "kmmlu_economics", "sample_len": 130, "acc,none": 0.27692307692307694, "acc_stderr,none": 0.03939825345266472 }, "kmmlu_education": { "name": "kmmlu_education", "alias": "kmmlu_education", "sample_len": 100, "acc,none": 0.28, "acc_stderr,none": 0.045126085985421296 }, "kmmlu_korean_history": { "name": "kmmlu_korean_history", "alias": "kmmlu_korean_history", "sample_len": 100, "acc,none": 0.23, "acc_stderr,none": 0.04229525846816507 }, "kmmlu_law": { "name": "kmmlu_law", "alias": "kmmlu_law", "sample_len": 1000, "acc,none": 0.24, "acc_stderr,none": 0.013512312258920847 }, "kmmlu_management": { "name": "kmmlu_management", "alias": "kmmlu_management", "sample_len": 1000, "acc,none": 0.204, "acc_stderr,none": 0.012749374359024311 }, "kmmlu_political_science_and_sociology": { "name": "kmmlu_political_science_and_sociology", "alias": "kmmlu_political_science_and_sociology", "sample_len": 300, "acc,none": 0.23, "acc_stderr,none": 0.024337372337779037 }, "kmmlu_psychology": { "name": "kmmlu_psychology", "alias": "kmmlu_psychology", "sample_len": 1000, "acc,none": 0.224, "acc_stderr,none": 0.013190830072364589 }, "kmmlu_social_welfare": { "name": "kmmlu_social_welfare", "alias": "kmmlu_social_welfare", "sample_len": 1000, "acc,none": 0.172, "acc_stderr,none": 0.011939788882495308 }, "kmmlu_taxation": { "name": "kmmlu_taxation", "alias": "kmmlu_taxation", "sample_len": 200, "acc,none": 0.235, "acc_stderr,none": 0.03005647949775547 }, "kobest": { "alias": "kobest", "name": "kobest", "sample_len": 4561, "acc,none": 0.5231308923481692, "acc_stderr,none": 0.007297298748221897, "acc_norm,none": 0.474, "acc_norm_stderr,none": 0.022352791650914174, "f1,none": 0.46754121249081004, "f1_stderr,none": "N/A", "sample_count": { "acc,none": 4561, "acc_norm,none": 500, "f1,none": 4561 } }, "kmmlu_stem": { "alias": "kmmlu_stem", "name": "kmmlu_stem", "sample_len": 9900, "acc,none": 0.13797979797979798, "acc_stderr,none": 0.003422447552202874, "sample_count": { "acc,none": 9900 } }, "kmmlu_other": { "alias": "kmmlu_other", "name": "kmmlu_other", "sample_len": 8400, "acc,none": 0.14952380952380953, "acc_stderr,none": 0.0038749065104715114, "sample_count": { "acc,none": 8400 } }, "kmmlu_applied_science": { "alias": "kmmlu_applied_science", "name": "kmmlu_applied_science", "sample_len": 11600, "acc,none": 0.12637931034482758, "acc_stderr,none": 0.0030704612706282304, "sample_count": { "acc,none": 11600 } }, "kmmlu_humss": { "alias": "kmmlu_humss", "name": "kmmlu_humss", "sample_len": 5130, "acc,none": 0.2155945419103314, "acc_stderr,none": 0.005734774245353025, "sample_count": { "acc,none": 5130 } }, "kmmlu": { "alias": "kmmlu", "name": "kmmlu", "sample_len": 35030, "acc,none": 0.1482729089351984, "acc_stderr,none": 0.001880917345299668, "sample_count": { "acc,none": 35030 } } }, "groups": { "kobest": { "alias": "kobest", "name": "kobest", "sample_len": 4561, "acc,none": 0.5231308923481692, "acc_stderr,none": 0.007297298748221897, "acc_norm,none": 0.474, "acc_norm_stderr,none": 0.022352791650914174, "f1,none": 0.46754121249081004, "f1_stderr,none": "N/A", "sample_count": { "acc,none": 4561, "acc_norm,none": 500, "f1,none": 4561 } }, "kmmlu_stem": { "alias": "kmmlu_stem", "name": "kmmlu_stem", "sample_len": 9900, "acc,none": 0.13797979797979798, "acc_stderr,none": 0.003422447552202874, "sample_count": { "acc,none": 9900 } }, "kmmlu_other": { "alias": "kmmlu_other", "name": "kmmlu_other", "sample_len": 8400, "acc,none": 0.14952380952380953, "acc_stderr,none": 0.0038749065104715114, "sample_count": { "acc,none": 8400 } }, "kmmlu_applied_science": { "alias": "kmmlu_applied_science", "name": "kmmlu_applied_science", "sample_len": 11600, "acc,none": 0.12637931034482758, "acc_stderr,none": 0.0030704612706282304, "sample_count": { "acc,none": 11600 } }, "kmmlu_humss": { "alias": "kmmlu_humss", "name": "kmmlu_humss", "sample_len": 5130, "acc,none": 0.2155945419103314, "acc_stderr,none": 0.005734774245353025, "sample_count": { "acc,none": 5130 } }, "kmmlu": { "alias": "kmmlu", "name": "kmmlu", "sample_len": 35030, "acc,none": 0.1482729089351984, "acc_stderr,none": 0.001880917345299668, "sample_count": { "acc,none": 35030 } } }, "group_subtasks": { "kobest": [ "kobest_boolq", "kobest_copa", "kobest_hellaswag", "kobest_sentineg", "kobest_wic" ], "kmmlu": [ "kmmlu_stem", "kmmlu_other", "kmmlu_applied_science", "kmmlu_humss" ], "kmmlu_stem": [ "kmmlu_biology", "kmmlu_chemical_engineering", "kmmlu_chemistry", "kmmlu_civil_engineering", "kmmlu_computer_science", "kmmlu_ecology", "kmmlu_electrical_engineering", "kmmlu_information_technology", "kmmlu_materials_engineering", "kmmlu_math", "kmmlu_mechanical_engineering" ], "kmmlu_other": [ "kmmlu_agricultural_sciences", "kmmlu_construction", "kmmlu_fashion", "kmmlu_food_processing", "kmmlu_health", "kmmlu_interior_architecture_and_design", "kmmlu_marketing", "kmmlu_patent", "kmmlu_public_safety", "kmmlu_real_estate", "kmmlu_refrigerating_machinery" ], "kmmlu_applied_science": [ "kmmlu_aviation_engineering_and_maintenance", "kmmlu_electronics_engineering", "kmmlu_energy_management", "kmmlu_environmental_science", "kmmlu_gas_technology_and_engineering", "kmmlu_geomatics", "kmmlu_industrial_engineer", "kmmlu_machine_design_and_manufacturing", "kmmlu_maritime_engineering", "kmmlu_nondestructive_testing", "kmmlu_railway_and_automotive_engineering", "kmmlu_telecommunications_and_wireless_technology" ], "kmmlu_humss": [ "kmmlu_accounting", "kmmlu_criminal_law", "kmmlu_economics", "kmmlu_education", "kmmlu_korean_history", "kmmlu_law", "kmmlu_management", "kmmlu_political_science_and_sociology", "kmmlu_psychology", "kmmlu_social_welfare", "kmmlu_taxation" ] }, "configs": { "kmmlu_accounting": { "task": "kmmlu_accounting", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Accounting", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_accounting.yaml" } }, "kmmlu_agricultural_sciences": { "task": "kmmlu_agricultural_sciences", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Agricultural-Sciences", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_agricultural_sciences.yaml" } }, "kmmlu_aviation_engineering_and_maintenance": { "task": "kmmlu_aviation_engineering_and_maintenance", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Aviation-Engineering-and-Maintenance", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_aviation_engineering_and_maintenance.yaml" } }, "kmmlu_biology": { "task": "kmmlu_biology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_biology.yaml" } }, "kmmlu_chemical_engineering": { "task": "kmmlu_chemical_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Chemical-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_chemical_engineering.yaml" } }, "kmmlu_chemistry": { "task": "kmmlu_chemistry", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_chemistry.yaml" } }, "kmmlu_civil_engineering": { "task": "kmmlu_civil_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Civil-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_civil_engineering.yaml" } }, "kmmlu_computer_science": { "task": "kmmlu_computer_science", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Computer-Science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_computer_science.yaml" } }, "kmmlu_construction": { "task": "kmmlu_construction", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Construction", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_construction.yaml" } }, "kmmlu_criminal_law": { "task": "kmmlu_criminal_law", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Criminal-Law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_criminal_law.yaml" } }, "kmmlu_ecology": { "task": "kmmlu_ecology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Ecology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_ecology.yaml" } }, "kmmlu_economics": { "task": "kmmlu_economics", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Economics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_economics.yaml" } }, "kmmlu_education": { "task": "kmmlu_education", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Education", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_education.yaml" } }, "kmmlu_electrical_engineering": { "task": "kmmlu_electrical_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Electrical-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_electrical_engineering.yaml" } }, "kmmlu_electronics_engineering": { "task": "kmmlu_electronics_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Electronics-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_electronics_engineering.yaml" } }, "kmmlu_energy_management": { "task": "kmmlu_energy_management", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Energy-Management", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_energy_management.yaml" } }, "kmmlu_environmental_science": { "task": "kmmlu_environmental_science", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Environmental-Science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_environmental_science.yaml" } }, "kmmlu_fashion": { "task": "kmmlu_fashion", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Fashion", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_fashion.yaml" } }, "kmmlu_food_processing": { "task": "kmmlu_food_processing", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Food-Processing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_food_processing.yaml" } }, "kmmlu_gas_technology_and_engineering": { "task": "kmmlu_gas_technology_and_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Gas-Technology-and-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_gas_technology_and_engineering.yaml" } }, "kmmlu_geomatics": { "task": "kmmlu_geomatics", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Geomatics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_geomatics.yaml" } }, "kmmlu_health": { "task": "kmmlu_health", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Health", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_health.yaml" } }, "kmmlu_industrial_engineer": { "task": "kmmlu_industrial_engineer", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Industrial-Engineer", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_industrial_engineer.yaml" } }, "kmmlu_information_technology": { "task": "kmmlu_information_technology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Information-Technology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_information_technology.yaml" } }, "kmmlu_interior_architecture_and_design": { "task": "kmmlu_interior_architecture_and_design", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Interior-Architecture-and-Design", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_interior_architecture_and_design.yaml" } }, "kmmlu_korean_history": { "task": "kmmlu_korean_history", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Korean-History", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_korean_history.yaml" } }, "kmmlu_law": { "task": "kmmlu_law", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_law.yaml" } }, "kmmlu_machine_design_and_manufacturing": { "task": "kmmlu_machine_design_and_manufacturing", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Machine-Design-and-Manufacturing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_machine_design_and_manufacturing.yaml" } }, "kmmlu_management": { "task": "kmmlu_management", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Management", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_management.yaml" } }, "kmmlu_maritime_engineering": { "task": "kmmlu_maritime_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Maritime-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_maritime_engineering.yaml" } }, "kmmlu_marketing": { "task": "kmmlu_marketing", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Marketing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_marketing.yaml" } }, "kmmlu_materials_engineering": { "task": "kmmlu_materials_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Materials-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_materials_engineering.yaml" } }, "kmmlu_math": { "task": "kmmlu_math", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Math", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_math.yaml" } }, "kmmlu_mechanical_engineering": { "task": "kmmlu_mechanical_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Mechanical-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_mechanical_engineering.yaml" } }, "kmmlu_nondestructive_testing": { "task": "kmmlu_nondestructive_testing", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Nondestructive-Testing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_nondestructive_testing.yaml" } }, "kmmlu_patent": { "task": "kmmlu_patent", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Patent", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_patent.yaml" } }, "kmmlu_political_science_and_sociology": { "task": "kmmlu_political_science_and_sociology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Political-Science-and-Sociology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_political_science_and_sociology.yaml" } }, "kmmlu_psychology": { "task": "kmmlu_psychology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_psychology.yaml" } }, "kmmlu_public_safety": { "task": "kmmlu_public_safety", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Public-Safety", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_public_safety.yaml" } }, "kmmlu_railway_and_automotive_engineering": { "task": "kmmlu_railway_and_automotive_engineering", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Railway-and-Automotive-Engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_railway_and_automotive_engineering.yaml" } }, "kmmlu_real_estate": { "task": "kmmlu_real_estate", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Real-Estate", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_real_estate.yaml" } }, "kmmlu_refrigerating_machinery": { "task": "kmmlu_refrigerating_machinery", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Refrigerating-Machinery", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_refrigerating_machinery.yaml" } }, "kmmlu_social_welfare": { "task": "kmmlu_social_welfare", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Social-Welfare", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_social_welfare.yaml" } }, "kmmlu_taxation": { "task": "kmmlu_taxation", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Taxation", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_taxation.yaml" } }, "kmmlu_telecommunications_and_wireless_technology": { "task": "kmmlu_telecommunications_and_wireless_technology", "dataset_path": "HAERAE-HUB/KMMLU", "dataset_name": "Telecommunications-and-Wireless-Technology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_target": "{{answer-1}}", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "dev", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답：", "doc_to_choice": [ "A", "B", "C", "D" ], "doc_to_target": "{{answer-1}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 2.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kmmlu\\default\\kmmlu_telecommunications_and_wireless_technology.yaml" } }, "kobest_boolq": { "task": "kobest_boolq", "dataset_path": "skt/kobest_v1", "dataset_name": "boolq", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "{{paragraph}} 질문: {{question}} 답변: ", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": [ "아니오", "예" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{paragraph}} 질문: {{question}} 답변: ", "doc_to_choice": [ "아니오", "예" ], "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "f1", "aggregation": "def macro_f1_score(items):\n from sklearn.metrics import f1_score\n\n unzipped_list = list(zip(*items))\n golds = unzipped_list[0]\n preds = unzipped_list[1]\n fscore = f1_score(golds, preds, average=\"macro\")\n return fscore\n", "average": "macro", "hf_evaluate": true, "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kobest\\kobest_boolq.yaml" } }, "kobest_copa": { "task": "kobest_copa", "dataset_path": "skt/kobest_v1", "dataset_name": "copa", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "def copa_doc_to_text(doc: dict) -> str:\n connector = {\"원인\": \" 왜냐하면\", \"결과\": \" 그래서\"}[doc[\"question\"].strip()]\n return f\"\"\"{doc[\"premise\"]} {connector}\"\"\"\n", "doc_to_target": "def copa_doc_to_target(doc: dict) -> str:\n correct_choice = doc[\"alternative_1\"] if doc[\"label\"] == 0 else doc[\"alternative_2\"]\n return f\"\"\"{correct_choice}\"\"\"\n", "unsafe_code": false, "doc_to_choice": "def copa_doc_to_choice(doc: dict) -> list:\n return [f\"\"\"{doc[\"alternative_1\"]}\"\"\", f\"\"\"{doc[\"alternative_2\"]}\"\"\"]\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "", "doc_to_target": "", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "f1", "aggregation": "def macro_f1_score(items):\n from sklearn.metrics import f1_score\n\n unzipped_list = list(zip(*items))\n golds = unzipped_list[0]\n preds = unzipped_list[1]\n fscore = f1_score(golds, preds, average=\"macro\")\n return fscore\n", "average": "macro", "hf_evaluate": true, "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kobest\\kobest_copa.yaml" } }, "kobest_hellaswag": { "task": "kobest_hellaswag", "dataset_path": "skt/kobest_v1", "dataset_name": "hellaswag", "training_split": "train", "validation_split": "validation", "test_split": "test", "process_docs": "def hellaswag_process_doc(doc: Dataset) -> Dataset:\n def preprocessor(dataset):\n return {\n \"query\": f\"\"\"문장: {dataset[\"context\"]}\"\"\",\n \"choices\": [\n dataset[\"ending_1\"],\n dataset[\"ending_2\"],\n dataset[\"ending_3\"],\n dataset[\"ending_4\"],\n ],\n \"gold\": int(dataset[\"label\"]),\n }\n\n return doc.map(preprocessor)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true }, { "metric": "f1", "aggregation": "def macro_f1_score(items):\n from sklearn.metrics import f1_score\n\n unzipped_list = list(zip(*items))\n golds = unzipped_list[0]\n preds = unzipped_list[1]\n fscore = f1_score(golds, preds, average=\"macro\")\n return fscore\n", "average": "macro", "hf_evaluate": true, "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kobest\\kobest_hellaswag.yaml" } }, "kobest_sentineg": { "task": "kobest_sentineg", "dataset_path": "skt/kobest_v1", "dataset_name": "sentineg", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "def sentineg_doc_to_text(doc: dict):\n return f\"\"\"문장: {doc[\"sentence\"]} 긍부정:\"\"\"\n", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": [ "부정", "긍정" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": [ "부정", "긍정" ], "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "f1", "aggregation": "def macro_f1_score(items):\n from sklearn.metrics import f1_score\n\n unzipped_list = list(zip(*items))\n golds = unzipped_list[0]\n preds = unzipped_list[1]\n fscore = f1_score(golds, preds, average=\"macro\")\n return fscore\n", "average": "macro", "hf_evaluate": true, "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kobest\\kobest_sentineg.yaml" } }, "kobest_wic": { "task": "kobest_wic", "dataset_path": "skt/kobest_v1", "dataset_name": "wic", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "def wic_doc_to_text(doc: dict) -> str:\n return f\"\"\"문장1: {doc[\"context_1\"]} 문장2: {doc[\"context_2\"]} 두 문장에서 {doc[\"word\"]}가 같은 뜻으로 쓰였나?\"\"\"\n", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": [ "아니오", "예" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": [ "아니오", "예" ], "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "f1", "aggregation": "def macro_f1_score(items):\n from sklearn.metrics import f1_score\n\n unzipped_list = list(zip(*items))\n golds = unzipped_list[0]\n preds = unzipped_list[1]\n fscore = f1_score(golds, preds, average=\"macro\")\n return fscore\n", "average": "macro", "hf_evaluate": true, "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024, "config_source": "C:\\Users\\wonye\\AppData\\Roaming\\Python\\Python314\\site-packages\\lm_eval\\tasks\\kobest\\kobest_wic.yaml" } } }, "versions": { "kmmlu": "2.0", "kmmlu_accounting": 2.0, "kmmlu_agricultural_sciences": 2.0, "kmmlu_applied_science": "2.0", "kmmlu_aviation_engineering_and_maintenance": 2.0, "kmmlu_biology": 2.0, "kmmlu_chemical_engineering": 2.0, "kmmlu_chemistry": 2.0, "kmmlu_civil_engineering": 2.0, "kmmlu_computer_science": 2.0, "kmmlu_construction": 2.0, "kmmlu_criminal_law": 2.0, "kmmlu_ecology": 2.0, "kmmlu_economics": 2.0, "kmmlu_education": 2.0, "kmmlu_electrical_engineering": 2.0, "kmmlu_electronics_engineering": 2.0, "kmmlu_energy_management": 2.0, "kmmlu_environmental_science": 2.0, "kmmlu_fashion": 2.0, "kmmlu_food_processing": 2.0, "kmmlu_gas_technology_and_engineering": 2.0, "kmmlu_geomatics": 2.0, "kmmlu_health": 2.0, "kmmlu_humss": "2.0", "kmmlu_industrial_engineer": 2.0, "kmmlu_information_technology": 2.0, "kmmlu_interior_architecture_and_design": 2.0, "kmmlu_korean_history": 2.0, "kmmlu_law": 2.0, "kmmlu_machine_design_and_manufacturing": 2.0, "kmmlu_management": 2.0, "kmmlu_maritime_engineering": 2.0, "kmmlu_marketing": 2.0, "kmmlu_materials_engineering": 2.0, "kmmlu_math": 2.0, "kmmlu_mechanical_engineering": 2.0, "kmmlu_nondestructive_testing": 2.0, "kmmlu_other": "2.0", "kmmlu_patent": 2.0, "kmmlu_political_science_and_sociology": 2.0, "kmmlu_psychology": 2.0, "kmmlu_public_safety": 2.0, "kmmlu_railway_and_automotive_engineering": 2.0, "kmmlu_real_estate": 2.0, "kmmlu_refrigerating_machinery": 2.0, "kmmlu_social_welfare": 2.0, "kmmlu_stem": "2.0", "kmmlu_taxation": 2.0, "kmmlu_telecommunications_and_wireless_technology": 2.0, "kobest": "1.0", "kobest_boolq": 1.0, "kobest_copa": 1.0, "kobest_hellaswag": 1.0, "kobest_sentineg": 1.0, "kobest_wic": 1.0 }, "n-shot": { "kmmlu_accounting": 0, "kmmlu_agricultural_sciences": 0, "kmmlu_applied_science": 0, "kmmlu_aviation_engineering_and_maintenance": 0, "kmmlu_biology": 0, "kmmlu_chemical_engineering": 0, "kmmlu_chemistry": 0, "kmmlu_civil_engineering": 0, "kmmlu_computer_science": 0, "kmmlu_construction": 0, "kmmlu_criminal_law": 0, "kmmlu_ecology": 0, "kmmlu_economics": 0, "kmmlu_education": 0, "kmmlu_electrical_engineering": 0, "kmmlu_electronics_engineering": 0, "kmmlu_energy_management": 0, "kmmlu_environmental_science": 0, "kmmlu_fashion": 0, "kmmlu_food_processing": 0, "kmmlu_gas_technology_and_engineering": 0, "kmmlu_geomatics": 0, "kmmlu_health": 0, "kmmlu_humss": 0, "kmmlu_industrial_engineer": 0, "kmmlu_information_technology": 0, "kmmlu_interior_architecture_and_design": 0, "kmmlu_korean_history": 0, "kmmlu_law": 0, "kmmlu_machine_design_and_manufacturing": 0, "kmmlu_management": 0, "kmmlu_maritime_engineering": 0, "kmmlu_marketing": 0, "kmmlu_materials_engineering": 0, "kmmlu_math": 0, "kmmlu_mechanical_engineering": 0, "kmmlu_nondestructive_testing": 0, "kmmlu_other": 0, "kmmlu_patent": 0, "kmmlu_political_science_and_sociology": 0, "kmmlu_psychology": 0, "kmmlu_public_safety": 0, "kmmlu_railway_and_automotive_engineering": 0, "kmmlu_real_estate": 0, "kmmlu_refrigerating_machinery": 0, "kmmlu_social_welfare": 0, "kmmlu_stem": 0, "kmmlu_taxation": 0, "kmmlu_telecommunications_and_wireless_technology": 0, "kobest": 0, "kobest_boolq": 0, "kobest_copa": 0, "kobest_hellaswag": 0, "kobest_sentineg": 0, "kobest_wic": 0 }, "higher_is_better": { "kmmlu_accounting": { "acc": true }, "kmmlu_agricultural_sciences": { "acc": true }, "kmmlu_applied_science": { "acc": true }, "kmmlu_aviation_engineering_and_maintenance": { "acc": true }, "kmmlu_biology": { "acc": true }, "kmmlu_chemical_engineering": { "acc": true }, "kmmlu_chemistry": { "acc": true }, "kmmlu_civil_engineering": { "acc": true }, "kmmlu_computer_science": { "acc": true }, "kmmlu_construction": { "acc": true }, "kmmlu_criminal_law": { "acc": true }, "kmmlu_ecology": { "acc": true }, "kmmlu_economics": { "acc": true }, "kmmlu_education": { "acc": true }, "kmmlu_electrical_engineering": { "acc": true }, "kmmlu_electronics_engineering": { "acc": true }, "kmmlu_energy_management": { "acc": true }, "kmmlu_environmental_science": { "acc": true }, "kmmlu_fashion": { "acc": true }, "kmmlu_food_processing": { "acc": true }, "kmmlu_gas_technology_and_engineering": { "acc": true }, "kmmlu_geomatics": { "acc": true }, "kmmlu_health": { "acc": true }, "kmmlu_humss": { "acc": true }, "kmmlu_industrial_engineer": { "acc": true }, "kmmlu_information_technology": { "acc": true }, "kmmlu_interior_architecture_and_design": { "acc": true }, "kmmlu_korean_history": { "acc": true }, "kmmlu_law": { "acc": true }, "kmmlu_machine_design_and_manufacturing": { "acc": true }, "kmmlu_management": { "acc": true }, "kmmlu_maritime_engineering": { "acc": true }, "kmmlu_marketing": { "acc": true }, "kmmlu_materials_engineering": { "acc": true }, "kmmlu_math": { "acc": true }, "kmmlu_mechanical_engineering": { "acc": true }, "kmmlu_nondestructive_testing": { "acc": true }, "kmmlu_other": { "acc": true }, "kmmlu_patent": { "acc": true }, "kmmlu_political_science_and_sociology": { "acc": true }, "kmmlu_psychology": { "acc": true }, "kmmlu_public_safety": { "acc": true }, "kmmlu_railway_and_automotive_engineering": { "acc": true }, "kmmlu_real_estate": { "acc": true }, "kmmlu_refrigerating_machinery": { "acc": true }, "kmmlu_social_welfare": { "acc": true }, "kmmlu_stem": { "acc": true }, "kmmlu_taxation": { "acc": true }, "kmmlu_telecommunications_and_wireless_technology": { "acc": true }, "kobest": { "acc": true, "f1": true, "acc_norm": true }, "kobest_boolq": { "acc": true, "f1": true }, "kobest_copa": { "acc": true, "f1": true }, "kobest_hellaswag": { "acc": true, "acc_norm": true, "f1": true }, "kobest_sentineg": { "acc": true, "f1": true }, "kobest_wic": { "acc": true, "f1": true } }, "n-samples": { "kobest_boolq": { "original": 1404, "effective": 1404 }, "kobest_copa": { "original": 1000, "effective": 1000 }, "kobest_hellaswag": { "original": 500, "effective": 500 }, "kobest_sentineg": { "original": 397, "effective": 397 }, "kobest_wic": { "original": 1260, "effective": 1260 }, "kmmlu_biology": { "original": 1000, "effective": 1000 }, "kmmlu_chemical_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_chemistry": { "original": 600, "effective": 600 }, "kmmlu_civil_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_computer_science": { "original": 1000, "effective": 1000 }, "kmmlu_ecology": { "original": 1000, "effective": 1000 }, "kmmlu_electrical_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_information_technology": { "original": 1000, "effective": 1000 }, "kmmlu_materials_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_math": { "original": 300, "effective": 300 }, "kmmlu_mechanical_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_agricultural_sciences": { "original": 1000, "effective": 1000 }, "kmmlu_construction": { "original": 1000, "effective": 1000 }, "kmmlu_fashion": { "original": 1000, "effective": 1000 }, "kmmlu_food_processing": { "original": 1000, "effective": 1000 }, "kmmlu_health": { "original": 100, "effective": 100 }, "kmmlu_interior_architecture_and_design": { "original": 1000, "effective": 1000 }, "kmmlu_marketing": { "original": 1000, "effective": 1000 }, "kmmlu_patent": { "original": 100, "effective": 100 }, "kmmlu_public_safety": { "original": 1000, "effective": 1000 }, "kmmlu_real_estate": { "original": 200, "effective": 200 }, "kmmlu_refrigerating_machinery": { "original": 1000, "effective": 1000 }, "kmmlu_aviation_engineering_and_maintenance": { "original": 1000, "effective": 1000 }, "kmmlu_electronics_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_energy_management": { "original": 1000, "effective": 1000 }, "kmmlu_environmental_science": { "original": 1000, "effective": 1000 }, "kmmlu_gas_technology_and_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_geomatics": { "original": 1000, "effective": 1000 }, "kmmlu_industrial_engineer": { "original": 1000, "effective": 1000 }, "kmmlu_machine_design_and_manufacturing": { "original": 1000, "effective": 1000 }, "kmmlu_maritime_engineering": { "original": 600, "effective": 600 }, "kmmlu_nondestructive_testing": { "original": 1000, "effective": 1000 }, "kmmlu_railway_and_automotive_engineering": { "original": 1000, "effective": 1000 }, "kmmlu_telecommunications_and_wireless_technology": { "original": 1000, "effective": 1000 }, "kmmlu_accounting": { "original": 100, "effective": 100 }, "kmmlu_criminal_law": { "original": 200, "effective": 200 }, "kmmlu_economics": { "original": 130, "effective": 130 }, "kmmlu_education": { "original": 100, "effective": 100 }, "kmmlu_korean_history": { "original": 100, "effective": 100 }, "kmmlu_law": { "original": 1000, "effective": 1000 }, "kmmlu_management": { "original": 1000, "effective": 1000 }, "kmmlu_political_science_and_sociology": { "original": 300, "effective": 300 }, "kmmlu_psychology": { "original": 1000, "effective": 1000 }, "kmmlu_social_welfare": { "original": 1000, "effective": 1000 }, "kmmlu_taxation": { "original": 200, "effective": 200 } }, "config": { "model": "hf", "model_args": { "pretrained": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\model", "tokenizer": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\runpod-checkpoints\\kawk500m-korean-instruct-v1\\checkpoint-00000739\\tokenizer", "dtype": "bfloat16", "max_length": 1024 }, "model_num_parameters": 505350400, "model_dtype": "torch.bfloat16", "model_revision": "main", "model_sha": "", "batch_size": 64, "batch_sizes": [], "device": "cuda:0", "use_cache": "C:\\Users\\wonye\\project\\AI\\KAWK50M\\artifacts\\benchmark-results\\20260809T050630Z\\kawk500m-instruct-v1\\response-cache.sqlite", "limit": null, "bootstrap_iters": 1000, "gen_kwargs": null, "random_seed": 1234, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": null, "date": 1786251990.405794, "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Home (10.0.26200 64鍮꾪듃)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.14.0 (tags/v3.14.0:ebf955d, Oct 7 2025, 10:15:03) [MSC v.1944 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-11-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 12.8.61\r\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5080\nNvidia driver version: 595.71\ncuDNN version: C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v12.8\\bin\\cudnn_ops_train64_8.dll\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nName: AMD Ryzen 7 7800X3D 8-Core Processor \nManufacturer: AuthenticAMD\nFamily: 107\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 4201\nMaxClockSpeed: 4201\nL2CacheSize: 8192\nL2CacheSpeed: None\nRevision: 24834\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] onnxruntime==1.28.0\n[pip3] torch==2.11.0+cu128\n[pip3] torchaudio==2.11.0+cu128\n[conda] Could not collect", "transformers_version": "4.57.3", "lm_eval_version": "0.4.12", "upper_git_hash": null, "tokenizer_pad_token": [ "", "3" ], "tokenizer_eos_token": [ "", "2" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 2, "max_length": 1024 } }