Deploy portal from RoboEval corl2026-challenge

Dockerfile

@@ -28,7 +28,7 @@ WORKDIR /home/user
 # otherwise reuse a stale clone) and fails the build if the tag has moved.
 ARG ROBOEVAL_REPO=https://github.com/Robo-Eval/RoboEval.git
 ARG ROBOEVAL_REF=corl2026-challenge
-ARG ROBOEVAL_SHA=f0705aa1676c0d97ac29bc7d981e707fa233ba17
+ARG ROBOEVAL_SHA=c8b79598c9fb9021652128020517d1847abe4c95
 RUN git clone --branch "$ROBOEVAL_REF" "$ROBOEVAL_REPO" RoboEval \
     && if [ -n "$ROBOEVAL_SHA" ]; then test "$(git -C RoboEval rev-parse HEAD)" = "$ROBOEVAL_SHA"; fi \
     && git -C RoboEval submodule update --init \

app.py

@@ -159,9 +159,11 @@ with gr.Blocks(title="RoboEval Challenge · CoRL 2026") as demo:
         "[Submission guidelines](https://robo-eval.github.io/challenge-submission.html)"
     )
     with gr.Tab("Leaderboard"):
-        gr.Markdown("Each team's most recent accepted submission, ranked by Success, then Task Progression, "
-                    "then submission time, until the composite score is published. ↑ higher is better, "
-                    "↓ lower is better. Every entry runs at 20 Hz; **Action** is the team's declared action mode.")
+        gr.Markdown("Each team's most recent accepted submission, ranked by the RoboEval Diagnostic Score (**SCORE**: "
+                    "40% outcome, 30% safety & stability, 20% efficiency, 10% coordination; "
+                    "[how it is computed](https://robo-eval.github.io/challenge-rules.html#ranking)), then Success, "
+                    "Task Progression, and submission time. ↑ higher is better, ↓ lower is better. Every entry runs "
+                    "at 20 Hz; **Action** is the team's declared action mode.")
         with gr.Row():
             task = gr.Dropdown(TASK_CHOICES, value="overall", label="Task")
             family = gr.Radio(FAMILY_CHOICES, value="All metrics", label="Metrics")

portal_core.py

@@ -23,7 +23,7 @@ from typing import Any, Dict, List, Optional, Tuple
 import requests
 import yaml
 
-from roboeval.challenge import spec
+from roboeval.challenge import scoring, spec
 from roboeval.challenge.package import MANIFEST
 from roboeval.challenge.validate import validate
 
@@ -457,6 +457,16 @@ def action_label(record: Dict[str, Any]) -> str:
     return f"{'EE' if mode.get('ee') else 'Joint'} {'absolute' if mode.get('absolute') else 'delta'}"
 
 
+FAMILY_SCORE_KEYS = {"Outcome": "outcome", "Efficiency": "efficiency", "Coordination": "coordination",
+                     "Safety & Stability": "safety"}
+
+
+def record_scores(record: Dict[str, Any], task: Optional[str]) -> Dict[str, float]:
+    """RDS and family scores for a submission, over the suite or one task."""
+    scores = scoring.suite_scores(record["summary"]["per_family"])
+    return scores["overall"] if task is None else scores["tasks"][task]
+
+
 def task_metrics(record: Dict[str, Any], task: Optional[str]) -> Dict[str, float]:
     """Suite means (task=None) or one task family's means for a submission."""
     summary = record["summary"]
@@ -464,18 +474,18 @@ def task_metrics(record: Dict[str, Any], task: Optional[str]) -> Dict[str, float
 
 
 def leaderboard(records: List[Dict[str, Any]], task: Optional[str] = None) -> List[Dict[str, Any]]:
-    """Most recent accepted submission per team, ranked by the tie-break order.
+    """Most recent accepted submission per team, over the suite or one task.
 
-    Until the normalized RoboEval Diagnostic Score is implemented, rows are
-    ordered by Success, then Task Progression, then earlier submission, over
-    the whole suite or over one task family.
+    Ranked by the RoboEval Diagnostic Score, then Success, Task Progression,
+    and earlier submission, as the rules' tie-break order specifies.
     """
     latest: Dict[str, Dict[str, Any]] = {}
     for r in sorted(records, key=lambda r: r["created_utc"]):
         if r["status"] in RANKED_STATUSES:
             latest[r["team_name"].casefold()] = r
     rows = [r for r in latest.values() if task is None or task in r["summary"].get("per_family", {})]
-    rows.sort(key=lambda r: (-task_metrics(r, task)["success"], -task_metrics(r, task)["tp"], r["created_utc"]))
+    rows.sort(key=lambda r: (-record_scores(r, task)["score"], -task_metrics(r, task)["success"],
+                             -task_metrics(r, task)["tp"], -record_scores(r, task)["safety"], r["created_utc"]))
     return rows
 
 
@@ -490,9 +500,11 @@ def leaderboard_table(records: List[Dict[str, Any]], observation_mode: Optional[
             continue
         if action_mode and action_label(r) != action_mode:
             continue
-        m = task_metrics(r, task)
+        m, sc = task_metrics(r, task), record_scores(r, task)
         row = {"Rank": len(out) + 1, "Team": r["team_name"], "Method": r["method_name"],
-               "Obs.": r["observation_mode"], "Action": action_label(r)}
+               "Obs.": r["observation_mode"], "Action": action_label(r), "SCORE ↑": round(sc["score"], 3)}
+        if family:
+            row[f"{family.upper()} SCORE ↑"] = round(sc[FAMILY_SCORE_KEYS[family]], 3)
         row.update({f"{k.upper()} {'↑' if k in HIGHER_IS_BETTER else '↓'}": round(m[k], 3) for k in keys})
         row["Open source"] = "yes" if r["open_source"] else ""
         row["Note"] = "organizer (not prize-eligible)" if r["organizer"] else ""

test_portal.py

@@ -94,6 +94,19 @@ def test_edited_file_breaks_manifest(cfg, store, tmp_path):
         core.process_submission(cfg, store, "alice", str(bad), sha(bad))
 
 
+def test_edited_summary_is_rejected(cfg, store, tmp_path):
+    def inflate(b):
+        r = json.loads(b)
+        r["summary"]["suite"]["success"] = 0.99
+        return json.dumps(r).encode()
+
+    bad = repack(Path(BUNDLE), tmp_path / "bad.tar.gz", edit={"submission/results.json": inflate},
+                 rewrite_manifest=True)
+    out = core.process_submission(cfg, store, "alice", str(bad), sha(bad))
+    assert out.record["status"] == core.REJECTED
+    assert any(i["check"] == "consistency" and "summary" in i["message"] for i in out.report)
+
+
 def test_edited_results_with_regenerated_manifest_is_rejected(cfg, store, tmp_path):
     def inflate(b):
         r = json.loads(b)
@@ -206,5 +219,5 @@ def test_leaderboard_views(cfg, store):
     per_task = core.leaderboard_table(records, task="LiftPot")
     assert per_task[0]["TL ↓"] == round(records[0]["summary"]["per_family"]["LiftPot"]["tl"], 3)
     coordination = core.leaderboard_table(records, family="Coordination")
-    assert {k for k in coordination[0] if "↑" in k or "↓" in k} == {"BAVD ↓", "BGVD ↓"}
+    assert {k for k in coordination[0] if "↑" in k or "↓" in k} == {"SCORE ↑", "COORDINATION SCORE ↑", "BAVD ↓", "BGVD ↓"}
     assert core.leaderboard_table(records, action_mode="Joint delta") == []

test_scoring.py

@@ -0,0 +1,42 @@
+"""Unit tests for the RoboEval Diagnostic Score (no simulator or bundle needed)."""
+import pytest
+
+from roboeval.challenge import scoring, spec
+
+REF = {"success": 1.0, "tp": 1.0, "cpl": 2.0, "jpl": 4.0, "opl": 3.0, "ct": 8.0, "tl": 160.0,
+       "bavd": 0.2, "bgvd": 0.05, "scc": 0.0, "ecc": 1.0, "sc": 0.0, "mjj": 20.0, "mcj": 5.0}
+
+
+@pytest.fixture(autouse=True)
+def reference(monkeypatch):
+    monkeypatch.setattr(scoring, "human_reference", lambda: {f.key: dict(REF) for f in spec.TASK_FAMILIES})
+
+
+def test_doing_nothing_scores_zero():
+    idle = {k: 0.0 for k in REF}          # no motion, no contacts, no success
+    assert scoring.task_scores("LiftPot", idle)["score"] == 0.0
+
+
+def test_human_level_success_scores_one():
+    assert scoring.task_scores("LiftPot", REF)["score"] == pytest.approx(1.0)
+
+
+def test_worse_behavior_lowers_score_smoothly():
+    m = dict(REF, cpl=4.0)                 # twice the human path length
+    assert scoring.metric_score("cpl", 4.0, 2.0) == pytest.approx(2.2 / 4.2)
+    assert scoring.task_scores("LiftPot", m)["efficiency"] < 1.0
+    assert scoring.task_scores("LiftPot", m)["score"] < 1.0
+
+
+def test_count_metric_with_zero_reference():
+    assert scoring.metric_score("scc", 1.0, 0.0) == pytest.approx(0.5)
+    assert scoring.metric_score("scc", 3.0, 0.0) == pytest.approx(0.25)
+
+
+def test_outcome_weighting_and_suite_mean():
+    half = dict(REF, success=0.5, tp=0.5)
+    t = scoring.task_scores("LiftPot", half)
+    assert t["outcome"] == pytest.approx(0.5)
+    assert t["score"] == pytest.approx(0.4 * 0.5 + 0.5 * 0.6)
+    s = scoring.suite_scores({"LiftPot": REF, "PackBox": half})
+    assert s["overall"]["score"] == pytest.approx((1.0 + t["score"]) / 2)