Spaces:
Running
Running
Size classes widen for large models (x1.25 up to 30M -> x1.5 from 100M)
Browse files- leaderboard.py +21 -7
leaderboard.py
CHANGED
|
@@ -10,9 +10,22 @@ import html
|
|
| 10 |
import math
|
| 11 |
|
| 12 |
MIN_GAMES_GLOW = 2 # a leader needs more than one ranked game, so a single lucky win doesn't glow
|
| 13 |
-
# Size classes are relative, not absolute
|
| 14 |
-
# (-20%
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
_CDN = "https://cdn-avatars.huggingface.co/v1/production/uploads/"
|
| 18 |
# owner on the Hub -> (display name, logo, "r, g, b", border colour). Same identities as the BananaMind leaderboard.
|
|
@@ -93,14 +106,14 @@ def leaders(entries, gap: int = 0) -> set:
|
|
| 93 |
"""Size-class leaders: the best Elo among rated models of similar relative size.
|
| 94 |
|
| 95 |
A model glows when (1) it has at least MIN_GAMES_GLOW ranked games, (2) its Elo is above the 1000 start,
|
| 96 |
-
(3) at least one other rated model
|
| 97 |
(4) none of those neighbours has a higher Elo. Ties glow together. `gap` is kept for API compatibility."""
|
| 98 |
out = set()
|
| 99 |
for e in entries:
|
| 100 |
p = e.get("params")
|
| 101 |
if not p or e.get("games", 0) < MIN_GAMES_GLOW or e["elo"] <= 1000:
|
| 102 |
continue
|
| 103 |
-
peers = [o for o in entries if o.get("params") and p
|
| 104 |
if len(peers) < 2:
|
| 105 |
continue # no neighbour to compare with
|
| 106 |
if all(o["elo"] <= e["elo"] for o in peers):
|
|
@@ -270,8 +283,9 @@ def leaderboard_html(entries, protocol: str, gap: int, gap_label: str) -> str:
|
|
| 270 |
return (
|
| 271 |
'<div class="lb-root">'
|
| 272 |
f'<section class="lb-panel lb-chart-panel">{views}'
|
| 273 |
-
f'<div class="lb-footer"><span><i class="lb-rainbow-dot"></i>Size-class leader: best Elo among models
|
| 274 |
-
f'(min. {MIN_GAMES_GLOW} ranked games, Elo above 1000)</span>
|
|
|
|
| 275 |
'<section class="lb-panel lb-param-panel"><div class="lb-heading"><div><h2>Elo vs. Parameters</h2>'
|
| 276 |
'<p class="lb-muted">Model size on a logarithmic scale · higher and further left is better</p></div></div>'
|
| 277 |
'<div class="lb-key"><span><i class="lb-quadrant-key"></i>Fewer parameters, higher Elo</span><span>┈ Pareto line</span></div>'
|
|
|
|
| 10 |
import math
|
| 11 |
|
| 12 |
MIN_GAMES_GLOW = 2 # a leader needs more than one ranked game, so a single lucky win doesn't glow
|
| 13 |
+
# Size classes are relative, not absolute. Two models are neighbours when the bigger one is at most `size_ratio`
|
| 14 |
+
# times the smaller one. The ratio is 1.25 (-20%/+25%) up to 30M and widens smoothly (log scale) to 1.5
|
| 15 |
+
# (-33%/+50%) from 100M on, so e.g. 50M vs 65M and 100M vs 135M share a class while tiny models stay fine-grained.
|
| 16 |
+
RATIO_SMALL, RATIO_LARGE = 1.25, 1.5
|
| 17 |
+
RATIO_FROM, RATIO_TO = 30e6, 100e6
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def size_ratio(p: float) -> float:
|
| 21 |
+
t = min(1.0, max(0.0, math.log10(p / RATIO_FROM) / math.log10(RATIO_TO / RATIO_FROM)))
|
| 22 |
+
return RATIO_SMALL + (RATIO_LARGE - RATIO_SMALL) * t
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def neighbours(a: float, b: float) -> bool:
|
| 26 |
+
"""Symmetric: the allowed ratio is taken at the pair's geometric-mean size."""
|
| 27 |
+
return max(a, b) / min(a, b) <= size_ratio(math.sqrt(a * b))
|
| 28 |
+
|
| 29 |
|
| 30 |
_CDN = "https://cdn-avatars.huggingface.co/v1/production/uploads/"
|
| 31 |
# owner on the Hub -> (display name, logo, "r, g, b", border colour). Same identities as the BananaMind leaderboard.
|
|
|
|
| 106 |
"""Size-class leaders: the best Elo among rated models of similar relative size.
|
| 107 |
|
| 108 |
A model glows when (1) it has at least MIN_GAMES_GLOW ranked games, (2) its Elo is above the 1000 start,
|
| 109 |
+
(3) at least one other rated model is a size neighbour (see `neighbours`), and
|
| 110 |
(4) none of those neighbours has a higher Elo. Ties glow together. `gap` is kept for API compatibility."""
|
| 111 |
out = set()
|
| 112 |
for e in entries:
|
| 113 |
p = e.get("params")
|
| 114 |
if not p or e.get("games", 0) < MIN_GAMES_GLOW or e["elo"] <= 1000:
|
| 115 |
continue
|
| 116 |
+
peers = [o for o in entries if o.get("params") and neighbours(p, o["params"])]
|
| 117 |
if len(peers) < 2:
|
| 118 |
continue # no neighbour to compare with
|
| 119 |
if all(o["elo"] <= e["elo"] for o in peers):
|
|
|
|
| 283 |
return (
|
| 284 |
'<div class="lb-root">'
|
| 285 |
f'<section class="lb-panel lb-chart-panel">{views}'
|
| 286 |
+
f'<div class="lb-footer"><span><i class="lb-rainbow-dot"></i>Size-class leader: best Elo among models of similar size '
|
| 287 |
+
f'(−20%/+25% up to 30M, widening to −33%/+50% from 100M · min. {MIN_GAMES_GLOW} ranked games, Elo above 1000)</span>'
|
| 288 |
+
f'<span>{len(entries)} models · click a model to open it ↗</span></div></section>'
|
| 289 |
'<section class="lb-panel lb-param-panel"><div class="lb-heading"><div><h2>Elo vs. Parameters</h2>'
|
| 290 |
'<p class="lb-muted">Model size on a logarithmic scale · higher and further left is better</p></div></div>'
|
| 291 |
'<div class="lb-key"><span><i class="lb-quadrant-key"></i>Fewer parameters, higher Elo</span><span>┈ Pareto line</span></div>'
|