|
|
| """Submission runner for Track A.
|
|
|
| This wrapper keeps the packaged output in the Phase 3 guideline shape:
|
|
|
| result/
|
| traces.json
|
| results.csv
|
| runtime.json
|
|
|
| It delegates prediction to the copied main.py, then normalizes filenames and
|
| columns. It does not train or download anything.
|
| """
|
|
|
| from __future__ import annotations
|
|
|
| import argparse
|
| import csv
|
| import json
|
| import os
|
| import shutil
|
| import subprocess
|
| import sys
|
| from pathlib import Path
|
|
|
|
|
| ROOT = Path(__file__).resolve().parent
|
|
|
|
|
| def _resolve(path_text: str) -> Path:
|
| path = Path(path_text)
|
| return path if path.is_absolute() else ROOT / path
|
|
|
|
|
| def _api_key_names(spec: str) -> list[str]:
|
| return [name.strip() for name in spec.split(",") if name.strip()]
|
|
|
|
|
| def _ensure_local_vllm_key(env: dict[str, str], spec: str) -> None:
|
| names = _api_key_names(spec)
|
| if names and not any(env.get(name) for name in names):
|
| env[names[0]] = "EMPTY"
|
|
|
|
|
| def _write_results_csv(raw_result_csv: Path, output_csv: Path) -> None:
|
| with raw_result_csv.open("r", encoding="utf-8", newline="") as f:
|
| reader = csv.DictReader(f)
|
| rows = list(reader)
|
|
|
| output_csv.parent.mkdir(parents=True, exist_ok=True)
|
| with output_csv.open("w", encoding="utf-8", newline="") as f:
|
| writer = csv.DictWriter(f, fieldnames=["scenario_id", "prediction"])
|
| writer.writeheader()
|
| for row in rows:
|
| scenario_id = row.get("scenario_id") or row.get("ID") or row.get("id") or ""
|
| prediction = row.get("prediction")
|
| if prediction is None:
|
| prediction = row.get("Track A", "")
|
| writer.writerow(
|
| {
|
| "scenario_id": str(scenario_id).strip(),
|
| "prediction": str(prediction or "").strip(),
|
| }
|
| )
|
|
|
|
|
| def _normalize_traces(traces_path: Path) -> list[dict]:
|
| if not traces_path.exists():
|
| raise FileNotFoundError(f"Missing traces file: {traces_path}")
|
|
|
| with traces_path.open("r", encoding="utf-8") as f:
|
| traces = json.load(f)
|
| if not isinstance(traces, list):
|
| raise ValueError("traces.json must contain a JSON list")
|
|
|
| normalized = []
|
| for i, row in enumerate(traces):
|
| if not isinstance(row, dict):
|
| row = {"completion": str(row)}
|
| row.setdefault("scenario_id", row.get("ID", f"scenario_{i + 1}"))
|
| row.setdefault("completion_id", 0)
|
| row.setdefault("completion", "")
|
| normalized.append(row)
|
|
|
| with traces_path.open("w", encoding="utf-8") as f:
|
| json.dump(normalized, f, indent=2, ensure_ascii=False)
|
| return normalized
|
|
|
|
|
| def _write_runtime_json(traces: list[dict], runtime_path: Path) -> None:
|
| runtimes = []
|
| for row in traces:
|
| runtime_seconds = row.get("runtime_seconds", row.get("execution_time_seconds", 0.0))
|
| try:
|
| runtime_seconds = float(runtime_seconds)
|
| except (TypeError, ValueError):
|
| runtime_seconds = 0.0
|
| runtimes.append(
|
| {
|
| "scenario_id": str(row.get("scenario_id", "")),
|
| "runtime_seconds": runtime_seconds,
|
| }
|
| )
|
|
|
| with runtime_path.open("w", encoding="utf-8") as f:
|
| json.dump(runtimes, f, indent=2)
|
|
|
|
|
| def parse_args() -> argparse.Namespace:
|
| parser = argparse.ArgumentParser()
|
| parser.add_argument(
|
| "--input",
|
| default=os.environ.get("TEST_PATH", "data/Phase_2/test.json"),
|
| help="Path to the Track A test JSON file.",
|
| )
|
| parser.add_argument(
|
| "--output",
|
| default="result",
|
| help="Directory where traces.json, results.csv, and runtime.json are written.",
|
| )
|
| parser.add_argument(
|
| "--model_bundle",
|
| default="models/model_v4_bundle.pkl",
|
| help="Path to the auxiliary Track A model bundle.",
|
| )
|
| parser.add_argument(
|
| "--server_url",
|
| default=os.environ.get("TRACK_A_SERVER_URL", "https://localhost:8081/no"),
|
| )
|
| parser.add_argument(
|
| "--model_url",
|
| default=os.environ.get("OPENAI_BASE_URL", "http://localhost:8001/v1"),
|
| )
|
| parser.add_argument(
|
| "--model_name",
|
| default=os.environ.get("OPENAI_MODEL", "Qwen3.5-35B-A3B"),
|
| )
|
| parser.add_argument(
|
| "--api_key_env",
|
| default="OPENAI_API_KEY,AGENT_API_KEY,OPENROUTER_API_KEY",
|
| )
|
| parser.add_argument("--concurrency", type=int, default=1)
|
| parser.add_argument("--max_steps", type=int, default=4)
|
| parser.add_argument("--max_tool_calls", type=int, default=8)
|
| parser.add_argument("--question_timeout", type=float, default=180.0)
|
| parser.add_argument("--checkpoint_every", type=int, default=1)
|
| parser.add_argument("--max_samples", type=int, default=None)
|
| parser.add_argument("--no_agent", action="store_true")
|
| args, extra = parser.parse_known_args()
|
| args.extra_main_args = extra
|
| return args
|
|
|
|
|
| def main() -> None:
|
| args = parse_args()
|
| input_path = _resolve(args.input)
|
| model_bundle = _resolve(args.model_bundle)
|
| output_dir = _resolve(args.output)
|
|
|
| if not input_path.exists():
|
| raise FileNotFoundError(f"Input test file not found: {input_path}")
|
| if not model_bundle.exists():
|
| raise FileNotFoundError(f"Model bundle not found: {model_bundle}")
|
|
|
| output_dir.mkdir(parents=True, exist_ok=True)
|
| work_dir = output_dir / "_raw_track_a"
|
| if work_dir.exists():
|
| shutil.rmtree(work_dir)
|
| work_dir.mkdir(parents=True)
|
|
|
| raw_result_csv = work_dir / "result.csv"
|
| debug_json = work_dir / "debug.json"
|
| traces_json = output_dir / "traces.json"
|
|
|
| command = [
|
| sys.executable,
|
| str(ROOT / "main.py"),
|
| "--test_path",
|
| str(input_path),
|
| "--model_bundle",
|
| str(model_bundle),
|
| "--track_b_test",
|
| "",
|
| "--out",
|
| str(raw_result_csv),
|
| "--debug_out",
|
| str(debug_json),
|
| "--traces_out",
|
| str(traces_json),
|
| "--server_url",
|
| args.server_url,
|
| "--model_url",
|
| args.model_url,
|
| "--model_name",
|
| args.model_name,
|
| "--api_key_env",
|
| args.api_key_env,
|
| "--concurrency",
|
| str(args.concurrency),
|
| "--max_steps",
|
| str(args.max_steps),
|
| "--max_tool_calls",
|
| str(args.max_tool_calls),
|
| "--question_timeout",
|
| str(args.question_timeout),
|
| "--checkpoint_every",
|
| str(args.checkpoint_every),
|
| "--no_progress",
|
| ]
|
| if args.max_samples is not None:
|
| command.extend(["--max_samples", str(args.max_samples)])
|
| if args.no_agent:
|
| command.append("--no_agent")
|
| command.extend(args.extra_main_args)
|
|
|
| env = os.environ.copy()
|
| _ensure_local_vllm_key(env, args.api_key_env)
|
| subprocess.run(command, cwd=str(ROOT), env=env, check=True)
|
|
|
| _write_results_csv(raw_result_csv, output_dir / "results.csv")
|
| traces = _normalize_traces(traces_json)
|
| _write_runtime_json(traces, output_dir / "runtime.json")
|
| shutil.rmtree(work_dir)
|
|
|
|
|
| if __name__ == "__main__":
|
| main()
|
|
|
