🎉 Reset repo

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/sync-to-hf-space.yml

@@ -0,0 +1,124 @@
+name: Sync main to HF Space
+
+# Mirrors every push to `main` on GitHub into the HF Space git remote so
+# that the public coding-agent-leaderboard Space (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard)
+# always tracks the source-of-truth repo.
+#
+# Required repository secrets (Settings -> Secrets and variables -> Actions):
+#   HF_TOKEN     Hugging Face access token with write permission to the Space.
+#                Create at https://huggingface.co/settings/tokens
+#                (token type "Write" is sufficient; no organization scope needed).
+#   HF_USERNAME  Optional fallback username if token introspection fails.
+#
+# Optional: set HF_SPACE_ID as a repo variable (not secret) to point the
+# workflow at a different Space; defaults to "taagarwa/coding-agent-leaderboard".
+
+on:
+  push:
+    branches: [main]
+  # Manual dispatch lets you re-mirror the latest main on demand from
+  # the Actions tab without pushing a new commit.
+  workflow_dispatch:
+
+# Only one mirror job at a time so we never race ourselves into
+# non-fast-forward pushes on the Space remote.
+concurrency:
+  group: sync-to-hf-space
+  cancel-in-progress: false
+
+jobs:
+  mirror:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout GitHub main (full history + LFS)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+
+      - name: Verify required secrets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          if [ -z "$HF_TOKEN" ]; then
+            echo "::error::HF_TOKEN repository secret must be set."
+            echo "  Create HF_TOKEN at https://huggingface.co/settings/tokens (type: Write)"
+            exit 1
+          fi
+
+      - name: Ensure HF Space exists
+        id: hf
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}
+        run: |
+          set -euo pipefail
+          python -m pip install --quiet 'huggingface_hub>=0.24,<2'
+          python - <<'PY'
+          import os
+
+          from huggingface_hub import HfApi
+
+          token = os.environ["HF_TOKEN"]
+          space_id = os.environ["HF_SPACE_ID"]
+          fallback_username = os.environ.get("HF_USERNAME", "").strip()
+
+          api = HfApi(token=token)
+          username = fallback_username
+          try:
+              info = api.whoami(token=token)
+              username = str(info.get("name") or username).strip()
+          except Exception as exc:
+              if not username:
+                  raise RuntimeError("HF_USERNAME fallback is required when token introspection fails") from exc
+
+          api.create_repo(
+              repo_id=space_id,
+              repo_type="space",
+              space_sdk="docker",
+              token=token,
+              exist_ok=True,
+          )
+
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+              output.write(f"username={username}\n")
+          print(f"HF Space ready: {space_id}")
+          PY
+
+      - name: Push to HF Space remote
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ steps.hf.outputs.username }}
+          HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}
+        run: |
+          set -euo pipefail
+          # Authenticate via token in the URL. HF Spaces accept the
+          # username + token basic-auth format over HTTPS git.
+          HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_SPACE_ID}"
+
+          # Configure identity for any metadata operations. The actual
+          # commits come from GitHub unchanged; we only push refs.
+          git config user.name  "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          echo "Pushing $(git rev-parse --short HEAD) to ${HF_SPACE_ID}..."
+
+          # --force is intentional: GitHub is the single source of truth
+          # for the Space's git history. Anything on the Space side that
+          # wasn't committed via GitHub is overwritten on the next sync.
+          # This prevents the drift situation where someone edits files
+          # in the HF Space UI and creates commits only visible there.
+          git push --force "${HF_REMOTE}" HEAD:main
+
+          echo "Sync complete."
+
+      - name: Summary
+        if: success()
+        run: |
+          echo "### HF Space mirror" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "Pushed \`$(git rev-parse --short HEAD)\` to \`${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}\` Space." >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "View the Space: <https://huggingface.co/spaces/${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}>" >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -0,0 +1,14 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/
+uv.lock

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,40 @@
+---
+title: Coding Agent Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+app_file: app.py
+pinned: true
+license: mit
+short_description: Compare coding agent models + harnesses
+sdk_version: 5.43.1
+tags:
+- leaderboard
+---
+
+# Coding Agent Leaderboard
+
+## Adding a New Leaderboard Entry
+
+Create a PR adding a new entry into the `results/` folder. 
+Check out `results/qwen3-6-35b-nvfp4-claude-code.json`(./results/qwen3-6-35b-nvfp4-claude-code.json) for an example result.
+
+## Development
+
+1. Install dependencies
+
+    ```sh
+    pip install -r requirements.txt
+
+    # or
+
+    uv venv
+    uv pip install -r requirements.txt
+    ```
+
+2. Run the app
+   
+    ```sh
+    python app.py
+    ```

app.py

@@ -0,0 +1,82 @@
+import os
+import re
+
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+
+from src.leaderboard import get_leaderboard_df, DISPLAY_BY_DEFAULT, SEARCH_COLUMNS
+from src.display.text_blocks import (
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+)
+
+REPO_ID = "taagarwa/coding-agent-leaderboard"
+TOKEN = os.environ.get("HF_TOKEN")
+API = HfApi(token=TOKEN)
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+
+LEADERBOARD_DF = get_leaderboard_df()
+
+def extract_body(s: str):
+    return re.match(r'\[(.*?)\]', s).group(1)
+
+
+def init_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    
+    # Make ColumnFilter choices from md format
+    dataset_choices = sorted({(extract_body(v), v) for v in dataframe["dataset"]})
+    
+    return Leaderboard(
+        value=dataframe,
+        select_columns=SelectColumns(
+            default_selection=DISPLAY_BY_DEFAULT,
+            label="Select Columns to Display:",
+        ),
+        datatype="markdown",
+        search_columns=SEARCH_COLUMNS,
+        filter_columns=[
+            ColumnFilter(label="Dataset", column="dataset", type="checkboxgroup", choices=dataset_choices),
+            ColumnFilter(label="Model License", column="model_license", type="checkboxgroup"),
+            ColumnFilter(label="Harness License", column="harness_license", type="checkboxgroup"),
+            ColumnFilter(label="Number of Parameters (B)", column="model_num_params", type="slider", min=0, max=4000, default=(0, 4000)),
+            ColumnFilter(label="Precision", column="precision", type="checkboxgroup"),
+        ],
+        interactive=False,
+    )
+
+demo = gr.Blocks()
+with demo:
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs() as tabs:
+        with gr.TabItem("🏅 Coding Agent Benchmark"):
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            gr.Markdown("\* `internal` refers to internal benchmarks performed by the model provider where the harness/environment were not made public")
+
+        with gr.TabItem("📝 About"):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

results/claude-opus-4-7-internal.json

@@ -0,0 +1,31 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "internal",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://www.anthropic.com/news/claude-opus-4-7"
+    },
+    "model": {
+        "name": "Claude Opus 4.7",
+        "repo": "Claude Opus 4.7",
+        "is_oss": false,
+        "num_params": 4000,
+        "precision": "bf16",
+        "url": "https://www.anthropic.com/news/claude-opus-4-7"
+    },
+    "environment": {
+        "name": "internal",
+        "url": "https://www.anthropic.com/news/claude-opus-4-7"
+    },
+    "metrics": {
+        "score": 0.876,
+        "time": null,
+        "costUSD": null
+    }
+}

results/qwen3-6-35b-internal.json

@@ -0,0 +1,31 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "internal",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://qwen.ai/blog?id=qwen3.6-35b-a3b"
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "Qwen/Qwen3.6-35B-A3B",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B"
+    },
+    "environment": {
+        "name": "internal",
+        "url": "https://qwen.ai/blog?id=qwen3.6-35b-a3b"
+    },
+    "metrics": {
+        "score": 0.734,
+        "time": null,
+        "costUSD": null
+    }
+}

results/qwen3-6-35b-nvfp4-claude-code.json

@@ -0,0 +1,44 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "claude-code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code"
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "nvfp4",
+        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "path": null,
+            "name": "swe-bench/swe-bench-verified",
+            "version": null,
+            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
+            "registry_url": null,
+            "registry_path": null,
+            "overwrite": false,
+            "download_dir": null,
+            "task_names": null,
+            "exclude_task_names": null,
+            "n_tasks": null
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "score": 0.632,
+        "time": 21600,
+        "costUSD": 48.00
+    }
+}

results/qwen3-6-35b-nvfp4-opencode.json

@@ -0,0 +1,45 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "opencode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode"
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "nvfp4",
+        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "path": null,
+            "name": "swe-bench/swe-bench-verified",
+            "version": null,
+            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
+            "registry_url": null,
+            "registry_path": null,
+            "overwrite": false,
+            "download_dir": null,
+            "task_names": null,
+            "exclude_task_names": null,
+            "n_tasks": null,
+            "accelerated_images": true
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "score": 0.548,
+        "time": 29940,
+        "costUSD": 66.53
+    }
+}

results/qwen3-6-36b-nvfp4-pi.json

@@ -0,0 +1,45 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/earendil-works/pi/tree/main"
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "nvfp4",
+        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "path": null,
+            "name": "swe-bench/swe-bench-verified",
+            "version": null,
+            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
+            "registry_url": null,
+            "registry_path": null,
+            "overwrite": false,
+            "download_dir": null,
+            "task_names": null,
+            "exclude_task_names": null,
+            "n_tasks": null,
+            "accelerated_images": true
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "score": 0.650,
+        "time": 23160,
+        "costUSD": 51.47
+    }
+}

src/display/text_blocks.py

@@ -0,0 +1,17 @@
+INTRODUCTION_TEXT = """
+Welcome to the Coding Agent Leaderboard!
+"""
+
+LLM_BENCHMARKS_TEXT = """
+## About
+
+Evaluate and compare Coding Agents.
+
+Coding Agent = Model + Harness + Skills.
+
+Visit our [GitHub repo](https://github.com/redhat-et/coding_agent_bench) for more details about the project.
+"""
+
+CITATION_BUTTON_TEXT = "TBD"
+
+CITATION_BUTTON_LABEL = "Citation"

src/leaderboard.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+import json
+import pandas as pd
+
+from src.models import Result
+
+RESULTS_DIR = Path(__file__).parent.parent / "results"
+
+DISPLAY_BY_DEFAULT = [
+    "dataset",
+    "model",
+    "precision",
+    "harness",
+    "skills",
+    "environment",
+    "score",
+]
+
+SEARCH_COLUMNS = [
+    "dataset",
+    "model",
+    "harness",
+]
+
+
+def format_time(seconds: int):
+    if seconds is None:
+        return None
+    m, s = divmod(seconds, 60)
+    h, m = divmod(m, 60)
+    return f"{h}h{m}m{s}s"
+
+
+def get_leaderboard_df():
+    results: list[Result] = []
+    for file in RESULTS_DIR.glob("*.json"):
+        with open(file, "r") as f:
+            data = json.load(f)
+        result = Result(**data)
+        results.append(result)
+
+    rows = []
+    for result in results:
+        rows.append(
+            {
+                "dataset": f'[{result.dataset.name}]({result.dataset.url})',
+                "model": result.model.name,
+                "model_id": f'[{result.model.repo}]({result.model.url})',
+                "precision": result.model.precision,
+                "harness": f'[{result.harness.name}]({result.harness.url})',
+                "skills": str(result.harness.skills) if result.harness.skills else "None",
+                "environment": f'[{result.environment.name}]({result.environment.url})',
+                "score": result.metrics.score,
+                "costUSD": result.metrics.costUSD,
+                "time": format_time(result.metrics.time),
+                "model_license": "FOSS" if result.model.is_oss else "Proprietary",
+                "harness_license": "FOSS" if result.harness.is_oss else "Proprietary",
+                "model_num_params": result.model.num_params,
+            }
+        )
+
+    leaderboard_df = pd.DataFrame(rows).sort_values(by="score", ascending=False)
+    return leaderboard_df

src/models.py

@@ -0,0 +1,46 @@
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+class Dataset(BaseModel):
+    name: str
+    repo: str
+    num_tasks: int
+    url: str
+
+
+class Harness(BaseModel):
+    name: str
+    skills: list[str]
+    is_oss: bool
+    url: str
+
+
+class Model(BaseModel):
+    name: str
+    repo: str | None = None
+    is_oss: bool
+    num_params: int
+    precision: str
+    url: str
+
+
+class Environment(BaseModel):
+    name: str
+    config: Optional[dict[str, Any]] = None
+    url: str
+
+
+class Metrics(BaseModel):
+    score: float
+    time: Optional[int] = None
+    costUSD: Optional[float] = None
+
+
+class Result(BaseModel):
+    dataset: Dataset
+    harness: Harness
+    model: Model
+    environment: Environment
+    metrics: Metrics