{
  "created": "2026-09-27T14:16:13Z",
  "concurrency": 1,
  "resumed": false,
  "new_run_wall_seconds": 3445.778244882007,
  "new_tasks": 20,
  "suites": {
    "gsm8k": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 275,
      "mean_input_tokens": 77.4,
      "mean_total_tokens": 352.4,
      "mean_model_seconds": 2.3928021708037703,
      "median_model_seconds": 2.220176461036317,
      "p95_model_seconds": 3.1514280069386587,
      "summed_request_seconds_per_correct": 2.3928021708037703,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "ifbench": {
      "attempted": 5,
      "correct": 3,
      "accuracy": 0.6,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 1952.6,
      "mean_input_tokens": 134.6,
      "mean_total_tokens": 2087.2,
      "mean_model_seconds": 21.503820910397916,
      "median_model_seconds": 10.035008723963983,
      "p95_model_seconds": 68.41221156402025,
      "summed_request_seconds_per_correct": 35.839701517329864,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "livecodebench": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 49329,
      "mean_input_tokens": 742.4,
      "mean_total_tokens": 50071.4,
      "mean_model_seconds": 660.0320765013806,
      "median_model_seconds": 480.73357990791555,
      "p95_model_seconds": 1812.1029692189768,
      "summed_request_seconds_per_correct": 660.0320765013806,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "tools": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 33.6,
      "mean_input_tokens": 330.6,
      "mean_total_tokens": 364.2,
      "mean_model_seconds": 0.6659889166243375,
      "median_model_seconds": 0.5145661529386416,
      "p95_model_seconds": 1.1192404899047688,
      "summed_request_seconds_per_correct": 0.6659889166243375,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    }
  },
  "quality_comparison_ready": true,
  "truncation_policy": "count_as_wrong",
  "truncated_task_ids": [],
  "suite_wall_seconds": 3445.778244882007,
  "wall_seconds_per_correct": 191.43212471566707
}