| { |
| "schema_version": 1, |
| "status": "complete_independently_audited", |
| "model": "New Open-Jev 27B on community-hard-mix-v2", |
| "optimizer_steps": 37160, |
| "checkpoint": { |
| "model": "Qwen/Qwen3.8-27B", |
| "method": "lora_decision_head", |
| "checkpoint_sha256": "c49994563c3c4f04a99d9130203c4e526f4ae5086c84deec57698d18cb652e71", |
| "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", |
| "temperature": 2.5343690298472983, |
| "code_commit": "7ac6bab261bd97efc3d446152999e94ae432e5b0", |
| "max_length": 16384 |
| }, |
| "scope": "231 public JevBench tasks only: 72 original, 48 easy, 111 hard. The private and judge tiers are unavailable; this is not the full 534-task benchmark and no full-benchmark composite is computed. ZefanCai Open-Jev uses Qwen LoRA weights and a decision head; it is a different system from the Kotoba and Codiv projects also called OpenJev.", |
| "benchmark": { |
| "upstream_commit": "f8ce71361165846101d02ebc83ad44e47ae44fc3", |
| "input_sha256": "2e20ff5f94dd04d015b3a7b9abd955757b6c5be25b55b97fcd141cb52f88b4aa", |
| "dataset_sha256": "dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51" |
| }, |
| "metrics": { |
| "n_planned": 231, |
| "n_attempted": 231, |
| "n_scorable": 231, |
| "n_valid": 231, |
| "n_correct": 197, |
| "accuracy": 0.8528138528138528, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "n_renormalized": 0, |
| "operational_success": 1.0, |
| "brier_mean": 0.2419831167860275, |
| "ordinal_mae": 0.3798003972534468 |
| }, |
| "per_public_tier": { |
| "original": { |
| "n_planned": 72, |
| "n_attempted": 72, |
| "n_valid": 72, |
| "n_correct": 69, |
| "accuracy": 0.9583333333333334 |
| }, |
| "easy": { |
| "n_planned": 48, |
| "n_attempted": 48, |
| "n_valid": 48, |
| "n_correct": 48, |
| "accuracy": 1.0 |
| }, |
| "hard": { |
| "n_planned": 111, |
| "n_attempted": 111, |
| "n_valid": 111, |
| "n_correct": 80, |
| "accuracy": 0.7207207207207207 |
| } |
| }, |
| "per_family": { |
| "policy": { |
| "n_planned": 12, |
| "n_correct": 11, |
| "accuracy": 0.9166666666666666 |
| }, |
| "intent": { |
| "n_planned": 24, |
| "n_correct": 23, |
| "accuracy": 0.9583333333333334 |
| }, |
| "ordinal": { |
| "n_planned": 12, |
| "n_correct": 12, |
| "accuracy": 1.0 |
| }, |
| "extraction": { |
| "n_planned": 24, |
| "n_correct": 23, |
| "accuracy": 0.9583333333333334 |
| }, |
| "adequacy": { |
| "n_planned": 12, |
| "n_correct": 12, |
| "accuracy": 1.0 |
| }, |
| "routing": { |
| "n_planned": 12, |
| "n_correct": 12, |
| "accuracy": 1.0 |
| }, |
| "fact": { |
| "n_planned": 12, |
| "n_correct": 12, |
| "accuracy": 1.0 |
| }, |
| "tool_selection": { |
| "n_planned": 12, |
| "n_correct": 12, |
| "accuracy": 1.0 |
| }, |
| "long_policy": { |
| "n_planned": 19, |
| "n_correct": 11, |
| "accuracy": 0.5789473684210527 |
| }, |
| "probability": { |
| "n_planned": 10, |
| "n_correct": 7, |
| "accuracy": 0.7 |
| }, |
| "temporal_numeric": { |
| "n_planned": 15, |
| "n_correct": 2, |
| "accuracy": 0.13333333333333333 |
| }, |
| "ambiguous": { |
| "n_planned": 7, |
| "n_correct": 5, |
| "accuracy": 0.7142857142857143 |
| }, |
| "multi_hop": { |
| "n_planned": 18, |
| "n_correct": 16, |
| "accuracy": 0.8888888888888888 |
| }, |
| "tradeoff": { |
| "n_planned": 6, |
| "n_correct": 5, |
| "accuracy": 0.8333333333333334 |
| }, |
| "adversarial": { |
| "n_planned": 6, |
| "n_correct": 6, |
| "accuracy": 1.0 |
| }, |
| "trap": { |
| "n_planned": 8, |
| "n_correct": 8, |
| "accuracy": 1.0 |
| }, |
| "judge_hard": { |
| "n_planned": 17, |
| "n_correct": 15, |
| "accuracy": 0.8823529411764706 |
| }, |
| "routing_hard": { |
| "n_planned": 5, |
| "n_correct": 5, |
| "accuracy": 1.0 |
| } |
| }, |
| "comparison_baselines": [ |
| { |
| "label": "Released Open-Jev 2B", |
| "overall": { |
| "n_correct": 150, |
| "n_planned": 231, |
| "accuracy": 0.6493506493506493 |
| }, |
| "hard": { |
| "n_correct": 46, |
| "n_planned": 111, |
| "accuracy": 0.4144144144144144 |
| } |
| }, |
| { |
| "label": "Released Open-Jev 9B", |
| "overall": { |
| "n_correct": 179, |
| "n_planned": 231, |
| "accuracy": 0.7748917748917749 |
| }, |
| "hard": { |
| "n_correct": 66, |
| "n_planned": 111, |
| "accuracy": 0.5945945945945946 |
| } |
| }, |
| { |
| "label": "Jev 1.13.0", |
| "overall": { |
| "n_correct": 200, |
| "n_planned": 231, |
| "accuracy": 0.8658008658008658 |
| }, |
| "hard": { |
| "n_correct": 81, |
| "n_planned": 111, |
| "accuracy": 0.7297297297297297 |
| } |
| } |
| ], |
| "protocol": { |
| "sum_tolerance_strict": 0.001, |
| "sum_tolerance_rounding": 0.02, |
| "argmax_tie_break": "lexicographically_smallest_label", |
| "score_accuracy": "argmax_level_equals_gold_level", |
| "probabilities": "native", |
| "concurrency": 1, |
| "retries": 0, |
| "warmups": 0, |
| "prefix_cache": false, |
| "latency_note": "Four-rank frozen-base FSDP2 execution on a shared node; one observation per task. Timing starts after dispatch and includes collective prediction, rank validation and response validation. No HTTP/network timing, warmups or retries. These diagnostics are not comparable to the old single-GPU HTTP latency, matched-hardware speedups, throughput or energy efficiency.", |
| "transport": "torch_distributed_collective", |
| "execution": { |
| "mode": "frozen_base_fsdp2", |
| "world_size": 4, |
| "backend": "nccl", |
| "node_shared_with_other_jobs": true, |
| "candidate_batch_size": 1, |
| "http_server_started": false |
| } |
| }, |
| "execution": { |
| "mode": "frozen_base_fsdp2", |
| "world_size": 4, |
| "backend": "nccl", |
| "node_shared_with_other_jobs": true, |
| "candidate_batch_size": 1, |
| "http_server_started": false |
| }, |
| "failed_requests": 0, |
| "pending_requests": 0, |
| "in_flight_requests": 0, |
| "audit": { |
| "file": "audit.json", |
| "sha256": "0759d6d9d457e0b823f823d8d9e7b7c9f0f4f3a9278f6cd32aac1e7b442392fc" |
| }, |
| "source_summary_sha256": "ede7fda4e114c22bed02ec5a389a3bd4b9952eeeeed4a64e39cdce88d6589b8f", |
| "limitations": [ |
| "231 public tasks only; unavailable private/judge tasks were not scored.", |
| "Four-rank shared-node direct-execution timings cannot be compared as speedups against the old single-GPU HTTP or hosted HTTPS timings.", |
| "Checkpoint hash verification establishes artifact identity; this offline audit does not rerun model inference or establish benchmark-training semantic separation.", |
| "Raw requests, labels, task IDs and responses remain in an ignored local run directory." |
| ] |
| } |
|
|
