{
  "created": "2026-09-27T13:16:04Z",
  "concurrency": 2,
  "resumed": false,
  "new_run_wall_seconds": 20949.800895433058,
  "new_tasks": 630,
  "suites": {
    "gsm8k": {
      "attempted": 200,
      "correct": 196,
      "accuracy": 0.98,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 380.925,
      "mean_input_tokens": 93.27,
      "mean_total_tokens": 474.195,
      "mean_model_seconds": 3.4486288186194725,
      "median_model_seconds": 2.997326802520547,
      "p95_model_seconds": 5.85073896299582,
      "summed_request_seconds_per_correct": 3.5190089985912985,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "ifbench": {
      "attempted": 300,
      "correct": 220,
      "accuracy": 0.7333333333333333,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 4998.316666666667,
      "mean_input_tokens": 125.44,
      "mean_total_tokens": 5123.756666666667,
      "mean_model_seconds": 55.58578871234068,
      "median_model_seconds": 27.256246656470466,
      "p95_model_seconds": 223.28189022897277,
      "summed_request_seconds_per_correct": 75.79880278955547,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "livecodebench": {
      "attempted": 100,
      "correct": 89,
      "accuracy": 0.89,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 1,
      "errors": 0,
      "truncated": 1,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 17271.87,
      "mean_input_tokens": 653.75,
      "mean_total_tokens": 17925.62,
      "mean_model_seconds": 243.10798836177565,
      "median_model_seconds": 61.351035209954716,
      "p95_model_seconds": 1182.111728347023,
      "summed_request_seconds_per_correct": 273.1550431031187,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "tools": {
      "attempted": 30,
      "correct": 28,
      "accuracy": 0.9333333333333333,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 40.333333333333336,
      "mean_input_tokens": 481.3333333333333,
      "mean_total_tokens": 521.6666666666666,
      "mean_model_seconds": 0.994276461203117,
      "median_model_seconds": 1.245341427042149,
      "p95_model_seconds": 1.324655186966993,
      "summed_request_seconds_per_correct": 1.065296208431911,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    }
  },
  "quality_comparison_ready": true,
  "truncation_policy": "count_as_wrong",
  "truncated_task_ids": [
    "lcb-atcoder-arc184_c"
  ],
  "suite_wall_seconds": 20949.800895433058,
  "wall_seconds_per_correct": 39.30544258054983
}