Upload app.py with huggingface_hub

app.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 
 import json
 import os
+import zipfile
+import tempfile
 from pathlib import Path
 
 import gradio as gr
@@ -46,11 +48,6 @@ MODEL_DISPLAY = {
     "gpt-5-nano": "GPT-5 Nano",
 }
 
-CONSISTENT_TEAM_HELPS = {
-    "D8_csv_cleanup", "MULTI1_fullstack_fix", "O2_incident_rootcause",
-    "P1_policy_config", "PIPE1_etl_fix", "TEST2_regression", "TEST5_mutation_resistant",
-}
-
 # ---------------------------------------------------------------------------
 # CSS
 # ---------------------------------------------------------------------------
@@ -62,7 +59,6 @@ CUSTOM_CSS = """
     --badge-radius: 4px;
 }
 
-/* Header */
 .tb-header {
     background: linear-gradient(135deg, #1e3a8a 0%, #1d4ed8 50%, #2563eb 100%);
     border-radius: 12px; padding: 28px 32px 24px; margin-bottom: 8px;
@@ -75,7 +71,6 @@ CUSTOM_CSS = """
     margin: 0; font-size: 1rem; color: #ffffff !important; opacity: 0.92;
 }
 
-/* Stat cards */
 .stat-grid {
     display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
     gap: 12px; margin-bottom: 16px;
@@ -88,19 +83,16 @@ CUSTOM_CSS = """
 .stat-card .stat-value { font-size: 1.5rem; font-weight: 700; color: var(--bench-blue); line-height: 1; }
 .stat-card .stat-label { font-size: 0.78rem; color: var(--bench-gray); margin-top: 4px; }
 
-/* Classification badges */
 .badge-HIGH-TNI   { background:#dbeafe; color:#1e40af; padding:2px 8px; border-radius:var(--badge-radius); font-size:0.78rem; font-weight:600; }
 .badge-TEAM-HELPS { background:#dcfce7; color:#166534; padding:2px 8px; border-radius:var(--badge-radius); font-size:0.78rem; font-weight:600; }
 .badge-NEUTRAL    { background:#f3f4f6; color:#374151; padding:2px 8px; border-radius:var(--badge-radius); font-size:0.78rem; font-weight:600; }
 .badge-TEAM-HURTS { background:#fee2e2; color:#991b1b; padding:2px 8px; border-radius:var(--badge-radius); font-size:0.78rem; font-weight:600; }
 
-/* Section headings */
 .section-title {
     font-size: 1.05rem; font-weight: 600; margin: 20px 0 10px;
     padding-bottom: 6px; border-bottom: 2px solid var(--border-color-primary, #e5e7eb);
 }
 
-/* About cards */
 .about-card {
     background: var(--background-fill-primary, #fff);
     border: 1px solid var(--border-color-primary, #e5e7eb);
@@ -110,12 +102,10 @@ CUSTOM_CSS = """
 .about-card p, .about-card li { font-size: 0.9rem; line-height: 1.65; }
 .about-card ul { padding-left: 20px; margin: 8px 0; }
 
-/* Role pills */
 .role-planner  { background:#ede9fe; color:#5b21b6; padding:3px 10px; border-radius:20px; font-size:0.82rem; font-weight:600; }
 .role-executor { background:#fef3c7; color:#92400e; padding:3px 10px; border-radius:20px; font-size:0.82rem; font-weight:600; }
 .role-verifier { background:#d1fae5; color:#065f46; padding:3px 10px; border-radius:20px; font-size:0.82rem; font-weight:600; }
 
-/* Submit section */
 .submit-info {
     background: #eff6ff; border: 1px solid #bfdbfe; border-radius: 10px;
     padding: 18px 22px; margin-bottom: 16px; font-size: 0.9rem; line-height: 1.7;
@@ -123,8 +113,13 @@ CUSTOM_CSS = """
 .submit-info code {
     background: #dbeafe; padding: 2px 6px; border-radius: 4px; font-size: 0.82rem;
 }
+.submit-step {
+    background: var(--background-fill-primary, #fff);
+    border: 1px solid var(--border-color-primary, #e5e7eb);
+    border-radius: 10px; padding: 16px 20px; margin-bottom: 12px;
+}
+.submit-step h4 { margin: 0 0 8px; font-size: 0.95rem; }
 
-/* Tables */
 .gr-dataframe table { width: 100%; font-size: 0.875rem; }
 .gr-dataframe th { font-weight: 600; }
 """
@@ -306,83 +301,172 @@ def build_category_summary_df(track: str = "Full") -> pd.DataFrame:
 
 
 # ---------------------------------------------------------------------------
-# Submission handler
+# Submission handler — workspace artifact zip (server-graded)
 # ---------------------------------------------------------------------------
 
-EXPECTED_TASKS = {t["task_id"] for t in _load()["per_task"]} if False else set()
+MAX_UPLOAD_MB = 100
 
 
-def _get_expected_tasks() -> set[str]:
+def _get_valid_task_ids() -> set[str]:
     return {t["task_id"] for t in _load()["per_task"]}
 
 
-def validate_submission(file, model_name: str, team_name: str, framework: str,
-                        description: str) -> str:
+def validate_and_accept_submission(
+    file, model_name: str, team_name: str, framework: str,
+    contact: str, description: str, seed: str,
+) -> str:
+    """Validate a workspace artifact zip and queue it for server-side grading."""
+
+    # --- basic field validation ---
     if not file:
-        return "Please upload a results JSON file."
+        return "Please upload a workspace artifact zip file."
     if not model_name or not model_name.strip():
         return "Please provide a model name."
+    if not team_name or not team_name.strip():
+        return "Please provide a team or organization name."
 
-    try:
-        with open(file.name) as f:
-            data = json.load(f)
-    except (json.JSONDecodeError, Exception) as e:
-        return f"Invalid JSON: {e}"
-
-    # Validate structure
-    if not isinstance(data, dict):
-        return "JSON root must be an object with 'model', 'results' keys."
-
-    results = data.get("results")
-    if not isinstance(results, list):
-        return "Missing 'results' array. See format specification below."
-
-    expected = _get_expected_tasks()
-    submitted_tasks = set()
-    errors = []
-
-    for i, r in enumerate(results):
-        if not isinstance(r, dict):
-            errors.append(f"results[{i}]: not an object")
-            continue
-        tid = r.get("task_id", "")
-        if not tid:
-            errors.append(f"results[{i}]: missing task_id")
-            continue
-        submitted_tasks.add(tid)
+    file_path = file.name if hasattr(file, "name") else str(file)
 
-        for cond in ["oracle", "restricted", "full", "team_no_plan", "team_no_verify"]:
-            val = r.get(cond)
-            if val is not None and not isinstance(val, (int, float)):
-                errors.append(f"results[{i}] ({tid}): {cond} must be a number")
+    # --- file size check ---
+    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+    if file_size_mb > MAX_UPLOAD_MB:
+        return f"File too large ({file_size_mb:.1f} MB). Maximum is {MAX_UPLOAD_MB} MB."
 
-    if errors:
-        return "Validation errors:\n" + "\n".join(errors[:20])
+    # --- zip validation ---
+    if not zipfile.is_zipfile(file_path):
+        return "Uploaded file is not a valid zip archive."
 
-    missing = expected - submitted_tasks
-    extra = submitted_tasks - expected
+    valid_tasks = _get_valid_task_ids()
+    found_tasks: dict[str, list[str]] = {}  # task_id -> list of files
+    has_meta = False
 
-    # Save submission
-    safe_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in model_name.strip())
-    out_path = SUBMISSIONS_DIR / f"{safe_name}.json"
-    submission = {
+    try:
+        with zipfile.ZipFile(file_path, "r") as zf:
+            names = zf.namelist()
+
+            # Security: reject paths with .. or absolute paths
+            for name in names:
+                if ".." in name or name.startswith("/"):
+                    return f"Rejected: unsafe path in zip: `{name}`"
+
+            # Check for meta.json
+            for name in names:
+                if name.endswith("meta.json") and name.count("/") <= 1:
+                    has_meta = True
+
+            # Find workspace directories. Accept multiple layouts:
+            #   tasks/{TASK_ID}/workspace/...
+            #   tasks/{TASK_ID}/{condition}/workspace/...
+            #   {TASK_ID}/workspace/...
+            #   {TASK_ID}/{condition}/workspace/...
+            valid_upper = {t.upper(): t for t in valid_tasks}
+
+            for name in names:
+                parts = name.split("/")
+                if "workspace" not in parts:
+                    continue
+                ws_idx = parts.index("workspace")
+                # Try task_id at ws_idx-1 or ws_idx-2
+                for offset in [1, 2]:
+                    candidate_idx = ws_idx - offset
+                    if candidate_idx < 0:
+                        continue
+                    candidate = parts[candidate_idx]
+                    matched = (
+                        candidate if candidate in valid_tasks
+                        else valid_upper.get(candidate.upper())
+                    )
+                    if matched:
+                        if matched not in found_tasks:
+                            found_tasks[matched] = []
+                        found_tasks[matched].append(name)
+                        break
+
+    except zipfile.BadZipFile:
+        return "Corrupted zip file. Please re-create and upload again."
+
+    if not found_tasks:
+        return (
+            "No valid task workspaces found in the zip.\n\n"
+            "Expected structure:\n"
+            "```\n"
+            "submission.zip/\n"
+            "  meta.json\n"
+            "  tasks/\n"
+            "    S1_hidden_spec/\n"
+            "      workspace/\n"
+            "        <files your agent produced>\n"
+            "    D1_schema_drift/\n"
+            "      workspace/\n"
+            "        ...\n"
+            "```"
+        )
+
+    # --- save submission ---
+    safe_model = "".join(c if c.isalnum() or c in "-_" else "_" for c in model_name.strip())
+    safe_team = "".join(c if c.isalnum() or c in "-_" else "_" for c in team_name.strip())
+    submission_id = f"{safe_team}__{safe_model}"
+
+    # Save metadata
+    meta = {
+        "submission_id": submission_id,
         "model": model_name.strip(),
-        "team": team_name.strip() if team_name else "",
+        "team": team_name.strip(),
         "framework": framework.strip() if framework else "",
+        "contact": contact.strip() if contact else "",
         "description": description.strip() if description else "",
-        "task_count": len(submitted_tasks),
-        "data": data,
+        "seed": int(seed) if seed and seed.isdigit() else 0,
+        "tasks_submitted": sorted(found_tasks.keys()),
+        "task_count": len(found_tasks),
+        "file_count": sum(len(v) for v in found_tasks.values()),
+        "status": "pending_grading",
     }
-    with open(out_path, "w") as f:
-        json.dump(submission, f, indent=2)
 
-    parts = [f"Submission received for **{model_name.strip()}** ({len(submitted_tasks)} tasks)."]
-    if missing:
-        parts.append(f"\n{len(missing)} tasks missing (partial submission OK).")
-    if extra:
-        parts.append(f"\n{len(extra)} unrecognized task IDs (ignored).")
-    parts.append("\nThank you! Your submission will be reviewed and added to the leaderboard.")
-    return "\n".join(parts)
+    meta_path = SUBMISSIONS_DIR / f"{submission_id}.json"
+    with open(meta_path, "w") as f:
+        json.dump(meta, f, indent=2)
+
+    # Save the zip itself
+    import shutil
+    zip_path = SUBMISSIONS_DIR / f"{submission_id}.zip"
+    shutil.copy2(file_path, zip_path)
+
+    # --- build response ---
+    missing = valid_tasks - set(found_tasks.keys())
+    conditions = ["oracle", "restricted", "full", "team_no_plan", "team_no_verify"]
+
+    response_parts = [
+        f"### Submission Received",
+        f"",
+        f"**Model**: {model_name.strip()}",
+        f"**Team**: {team_name.strip()}",
+        f"**Tasks**: {len(found_tasks)} / {len(valid_tasks)}",
+        f"**Files**: {sum(len(v) for v in found_tasks.values())} workspace artifacts",
+        f"**Seed**: {meta['seed']}",
+    ]
+
+    if framework and framework.strip():
+        response_parts.append(f"**Framework**: {framework.strip()}")
+
+    response_parts.append("")
+
+    if len(found_tasks) < len(valid_tasks):
+        response_parts.append(
+            f"*{len(missing)} tasks not included (partial submission accepted).*"
+        )
+
+    response_parts += [
+        "",
+        "**Status: Queued for server-side grading.**",
+        "",
+        "Your workspace artifacts will be graded against our hidden test suites. "
+        "Scores are computed server-side — not self-reported — to ensure benchmark integrity.",
+        "",
+        "Results will appear on the leaderboard once grading completes (typically within 24-48 hours). "
+        "You will be contacted at the provided email if there are any issues.",
+    ]
+
+    return "\n".join(response_parts)
 
 
 # ---------------------------------------------------------------------------
@@ -594,47 +678,71 @@ def build_app() -> gr.Blocks:
             with gr.Tab("Submit"):
                 gr.HTML("""
 <div class="submit-info">
-  <h3 style="margin:0 0 10px;">Submit Your Results</h3>
-  <p>Run the TeamBench evaluation locally and upload your results to appear on the leaderboard.</p>
-
-  <strong>Step 1: Install &amp; Run</strong>
-  <pre style="background:#dbeafe; padding:10px 14px; border-radius:6px; margin:8px 0; font-size:0.82rem; overflow-x:auto;">pip install teambench
-teambench run --model your-model-name --output results.json</pre>
-
-  <strong>Step 2: Upload</strong>
-  <p>Upload the generated <code>results.json</code> file below.</p>
-
-  <strong>Expected JSON format:</strong>
-  <pre style="background:#dbeafe; padding:10px 14px; border-radius:6px; margin:8px 0; font-size:0.82rem; overflow-x:auto;">{
-  "model": "your-model-name",
-  "framework": "langgraph",
-  "results": [
-    {
-      "task_id": "S1_hidden_spec",
-      "oracle": 1.0,
-      "restricted": 0.0,
-      "full": 0.75,
-      "team_no_plan": 0.5,
-      "team_no_verify": 0.75
-    },
-    ...
-  ]
-}</pre>
+  <h3 style="margin:0 0 12px;">Submit Your Results</h3>
+  <p>TeamBench uses <strong>server-side grading</strong> to ensure benchmark integrity.
+  You submit workspace artifacts (the files your agent produced) &mdash;
+  not self-reported scores. We grade them against hidden test suites.</p>
+</div>
+""")
+
+                gr.HTML("""
+<div class="submit-step">
+  <h4>Step 1: Install and Run</h4>
+  <pre style="background:#f3f4f6; padding:10px 14px; border-radius:6px; font-size:0.82rem; overflow-x:auto; margin:8px 0;">pip install teambench
+teambench run --model your-model --seed 0 --output submission/</pre>
+  <p style="font-size:0.85rem; color:#6b7280; margin:4px 0 0;">
+    This runs your agent on all tasks and saves the workspace artifacts (files your agent wrote).
+    You must run all 5 conditions: <code>oracle</code>, <code>restricted</code>, <code>full</code>,
+    <code>team_no_plan</code>, <code>team_no_verify</code>.
+  </p>
+</div>
+
+<div class="submit-step">
+  <h4>Step 2: Package</h4>
+  <pre style="background:#f3f4f6; padding:10px 14px; border-radius:6px; font-size:0.82rem; overflow-x:auto; margin:8px 0;">teambench package submission/ --output submission.zip</pre>
+  <p style="font-size:0.85rem; color:#6b7280; margin:4px 0 0;">
+    Creates a zip with the required structure. Partial submissions (subset of 155 tasks) are accepted.
+  </p>
+</div>
 
-  <p>All 5 conditions (<code>oracle</code>, <code>restricted</code>, <code>full</code>,
-  <code>team_no_plan</code>, <code>team_no_verify</code>) are expected per task.
-  Partial submissions (subset of 155 tasks) are accepted.</p>
+<div class="submit-step">
+  <h4>Step 3: Upload</h4>
+  <p style="font-size:0.85rem; color:#6b7280;">
+    Fill in the fields below and upload your <code>submission.zip</code>.
+    Results appear on the leaderboard after server-side grading (24-48 hours).
+  </p>
 </div>
+""")
+
+                gr.HTML("""
+<details style="margin-bottom:16px;">
+  <summary style="cursor:pointer; font-weight:600; font-size:0.9rem; padding:8px 0;">
+    Expected zip structure
+  </summary>
+  <pre style="background:#f3f4f6; padding:12px 16px; border-radius:6px; font-size:0.82rem; margin-top:8px;">submission.zip/
+  meta.json                        # auto-generated by teambench package
+  tasks/
+    S1_hidden_spec/
+      oracle/workspace/            # workspace after oracle condition
+      restricted/workspace/        # workspace after restricted condition
+      full/workspace/              # workspace after full team condition
+      team_no_plan/workspace/      # workspace after no-plan condition
+      team_no_verify/workspace/    # workspace after no-verify condition
+    D1_schema_drift/
+      oracle/workspace/
+      ...
+    ...</pre>
+</details>
 """)
 
                 with gr.Row():
                     model_input = gr.Textbox(
-                        label="Model Name",
+                        label="Model Name *",
                         placeholder="e.g. gpt-5, claude-opus-4, gemini-3-pro",
                         scale=2,
                     )
                     team_input = gr.Textbox(
-                        label="Team / Organization",
+                        label="Team / Organization *",
                         placeholder="e.g. OpenAI, Google DeepMind, your-lab-name",
                         scale=2,
                     )
@@ -649,24 +757,33 @@ teambench run --model your-model-name --output results.json</pre>
                         placeholder="your@email.com",
                         scale=2,
                     )
-
-                description_input = gr.Textbox(
-                    label="Description (optional)",
-                    placeholder="Brief description of your agent setup, tools used, etc.",
-                    lines=2,
-                )
+                with gr.Row():
+                    description_input = gr.Textbox(
+                        label="Description (optional)",
+                        placeholder="Brief description of your agent setup, tools used, etc.",
+                        lines=2,
+                        scale=3,
+                    )
+                    seed_input = gr.Textbox(
+                        label="Seed",
+                        value="0",
+                        placeholder="0",
+                        scale=1,
+                    )
 
                 file_input = gr.File(
-                    label="Upload results.json",
-                    file_types=[".json"],
+                    label="Upload submission.zip (workspace artifacts, max 100 MB)",
+                    file_types=[".zip"],
                 )
 
-                submit_btn = gr.Button("Submit Results", variant="primary")
+                submit_btn = gr.Button("Submit for Grading", variant="primary")
                 result_output = gr.Markdown("")
 
                 submit_btn.click(
-                    fn=validate_submission,
-                    inputs=[file_input, model_input, team_input, framework_input, description_input],
+                    fn=validate_and_accept_submission,
+                    inputs=[file_input, model_input, team_input,
+                            framework_input, contact_input,
+                            description_input, seed_input],
                     outputs=result_output,
                 )