{
  "created": "2026-09-26T21:13:49Z",
  "concurrency": 1,
  "resumed": false,
  "new_run_wall_seconds": 4374.499300124997,
  "new_tasks": 20,
  "suites": {
    "gsm8k": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 275.2,
      "mean_input_tokens": 77.4,
      "mean_total_tokens": 352.6,
      "mean_model_seconds": 2.2795148026081735,
      "median_model_seconds": 2.2747438669903204,
      "p95_model_seconds": 2.7609281560289674,
      "summed_request_seconds_per_correct": 2.2795148026081735,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "ifbench": {
      "attempted": 5,
      "correct": 4,
      "accuracy": 0.8,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 5641,
      "mean_input_tokens": 134.6,
      "mean_total_tokens": 5775.6,
      "mean_model_seconds": 60.683199993602464,
      "median_model_seconds": 33.55660462501692,
      "p95_model_seconds": 188.0719450309989,
      "summed_request_seconds_per_correct": 75.85399999200308,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "livecodebench": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 63239.6,
      "mean_input_tokens": 742.4,
      "mean_total_tokens": 63982,
      "mean_model_seconds": 807.4264700495871,
      "median_model_seconds": 839.0481605409877,
      "p95_model_seconds": 1524.661622893007,
      "summed_request_seconds_per_correct": 807.4264700495871,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    },
    "tools": {
      "attempted": 5,
      "correct": 5,
      "accuracy": 1.0,
      "truncation_policy": "count_as_wrong",
      "truncated_counted_as_wrong": 0,
      "errors": 0,
      "truncated": 0,
      "incomplete_token_counts": 0,
      "mean_output_tokens": 32.8,
      "mean_input_tokens": 330.6,
      "mean_total_tokens": 363.4,
      "mean_model_seconds": 0.6404867220087909,
      "median_model_seconds": 0.4934498239890672,
      "p95_model_seconds": 1.0912616860005073,
      "summed_request_seconds_per_correct": 0.6404867220087909,
      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
    }
  },
  "quality_comparison_ready": true,
  "truncation_policy": "count_as_wrong",
  "truncated_task_ids": [],
  "suite_wall_seconds": 4374.499300124997,
  "wall_seconds_per_correct": 230.23680526973666
}