File size: 2,776 Bytes
13e1e16 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | {
"schema": "br-voice-reasoner-evaluation-protocol.v1",
"evaluation_date_utc": "2026-09-02",
"benchmark": {
"name": "VoiceBench",
"upstream_repository": "https://github.com/matthewcym/VoiceBench",
"snapshot_identity": "SHA256 manifest",
"git_revision": null,
"revision_note": "The evaluated local snapshot did not retain Git metadata; release-relevant files are identified by SHA256."
},
"subset_samples": {
"wildvoice": 1000,
"bbh": 1000,
"alpacaeval_full": 636,
"mmsu": 3074,
"openbookqa": 455,
"ifeval": 345,
"advbench": 520,
"commoneval": 200,
"sdqa_usa": 553,
"total": 7783
},
"generation": {
"reasoning_enabled": true,
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"maximum_tokens": 16384,
"seed_policy": "A deterministic per-sample primary seed is derived from the input and base seed 1234.",
"empty_final_policy": {
"primary_attempts": 3,
"maximum_consecutive_fallback_seeds": 8,
"selection": "First non-empty visible final answer; no ranking or best-of selection.",
"samples_entering_seed_fallback": 5,
"samples_recovered": 4,
"samples_exhausted": 1,
"exhausted_record": "No answer.",
"exhausted_scoring": "incorrect"
}
},
"model_judge": {
"requested_model": "openai/gpt-4o-mini",
"model_kind": "alias",
"accessed_utc": "2026-09-02",
"gateway": "OpenRouter",
"provider_only": "OpenAI",
"allow_provider_fallbacks": false,
"judgments_per_sample": 3,
"vote_mode": "three separate requests without a seed",
"temperature": 0.5,
"top_p": 0.95,
"maximum_tokens": 1024
},
"aggregation": "Open-ended ratings are normalized to 0-100. Overall is the arithmetic mean of nine 0-100 subset scores.",
"sha256": {
"main.py": "cb7c86d7149b76d50e3316d09a377fb7cfa6549855ad2922f86b7dd980416f3e",
"evaluate.py": "0b610247b1373f2c256a4f183c2dfcbe7c562b576823cf4ce1396bf036ba1f42",
"src/models/qwen3_omni_thinking.py": "f44bcbc3c9b866c13b4db33ff156026dcf136f7fcfbd223ea006e600986543d8",
"qwen3omni_thinking/run_voicebench.sh": "4f040e0f695a12ee6a9fe4e1164c6fe3e57582160d5afca99d975f6ce7a7226c",
"qwen3omni_thinking/audit_outputs.py": "3ae9335dff1d568d19dc395bdf2b352b6a81685a9128edccdeca3882bdee7fb0",
"qwen3omni_thinking/score_local.sh": "332d701640ee1d76b3d7566e7250c55055546b3aa8186cd443531886635511ac",
"qwen3omni_thinking/score_thinking_mcq.py": "505018a7ae166f71f1a1d19b99887f69adc7fc802f91ceca930dbf2349f858eb",
"qwen3omni_thinking/api_judge_openrouter.py": "5b721566c1095e5dcf83b909f6ea667d127aed2533315b73f0819d168071ec33",
"qwen3omni_thinking/repro_utils.py": "9b36582a40c858f9ae6f5c64f7230dd7cf7db2e868d4f2bd77c446e70ea1a046"
}
}
|