{ "results": { "hellaswag": { "name": "hellaswag", "alias": "hellaswag", "sample_len": 100, "acc,none": 0.44, "acc_stderr,none": 0.049888765156985884, "acc_norm,none": 0.62, "acc_norm_stderr,none": 0.04878317312145634 }, "arc_easy": { "name": "arc_easy", "alias": "arc_easy", "sample_len": 100, "acc,none": 0.71, "acc_stderr,none": 0.045604802157206865, "acc_norm,none": 0.73, "acc_norm_stderr,none": 0.04461960433384737 } }, "group_subtasks": {}, "configs": { "arc_easy": { "task": "arc_easy", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "pretrained": "models/Qwen/Qwen3.5-4B-A3B-Student-v2", "dtype": "bfloat16", "config_source": "/Volumes/외장디스크1/오픈소스 모델/research/qwen35-moe-a3b/.venv-eval/lib/python3.14/site-packages/lm_eval/tasks/arc/arc_easy.yaml" } }, "hellaswag": { "task": "hellaswag", "dataset_path": "Rowan/hellaswag", "training_split": "train", "validation_split": "validation", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "models/Qwen/Qwen3.5-4B-A3B-Student-v2", "dtype": "bfloat16", "config_source": "/Volumes/외장디스크1/오픈소스 모델/research/qwen35-moe-a3b/.venv-eval/lib/python3.14/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml" } } }, "versions": { "arc_easy": 1.0, "hellaswag": 1.0 }, "n-shot": { "arc_easy": 0, "hellaswag": 0 }, "higher_is_better": { "arc_easy": { "acc": true, "acc_norm": true }, "hellaswag": { "acc": true, "acc_norm": true } }, "n-samples": { "hellaswag": { "original": 10042, "effective": 100 }, "arc_easy": { "original": 2376, "effective": 100 } }, "config": { "model": "hf", "model_args": { "pretrained": "models/Qwen/Qwen3.5-4B-A3B-Student-v2", "dtype": "bfloat16" }, "model_num_parameters": 3995901760, "model_dtype": "torch.bfloat16", "model_revision": "main", "model_sha": "", "batch_size": "1", "batch_sizes": [], "device": "mps", "use_cache": null, "limit": 100.0, "bootstrap_iters": 100000, "gen_kwargs": {}, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": null, "date": 1787549803.098937, "pretty_env_info": "PyTorch version: 2.11.0\nIs debug build: False\nCUDA used to build PyTorch: None\nROCM used to build PyTorch: N/A\n\nOS: macOS 26.4.1 (arm64)\nGCC version: Could not collect\nClang version: 21.0.0 (clang-2100.0.123.102)\nCMake version: version 4.4.0\nLibc version: N/A\n\nPython version: 3.14.6 (main, Jun 10 2026, 10:03:53) [Clang 21.0.0 (clang-2100.0.123.102)] (64-bit runtime)\nPython platform: macOS-26.4.1-arm64-arm-64bit-Mach-O\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nApple M4\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.4\n[pip3] torch==2.11.0\n[conda] Could not collect", "transformers_version": "5.13.0", "lm_eval_version": "0.4.12", "upper_git_hash": null, "tokenizer_pad_token": [ "<|endoftext|>", "248044" ], "tokenizer_eos_token": [ "<|im_end|>", "248046" ], "tokenizer_bos_token": [ null, "None" ], "eot_token_id": 248046, "max_length": 262144, "task_hashes": { "hellaswag": "4f7d86a1e256013e93651a0d8510973180c5f31142ca184d9677955a29676af1", "arc_easy": "fd3a493579cfdccf229a32d7c26710006fc4edf7c024add74f70c416c6dab3ce" }, "model_source": "hf", "model_name": "models/Qwen/Qwen3.5-4B-A3B-Student-v2", "model_name_sanitized": "models__Qwen__Qwen3.5-4B-A3B-Student-v2", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": null, "chat_template": null, "chat_template_sha": null, "total_evaluation_time_seconds": "244.92804725002497" }