Only 22 models (incl. BackKiyo-10M); 100M+ models play each other freely

app.py

@@ -30,28 +30,22 @@ RESULTS_REPO = os.environ.get("RESULTS_REPO", "DedeProGames/lm-tetris-arena-resu
 MAX_MODELS = int(os.environ.get("MAX_MODELS", 4))
 # The random ranked pick only groups models within this size gap, so big models don't farm Elo from tiny ones
 MAX_PARAM_GAP = int(os.environ.get("MAX_PARAM_GAP", 20_000_000))
+# ...except big models: when every model has at least this many parameters, any of them can play each other
+LARGE_FROM = int(os.environ.get("LARGE_FROM", 100_000_000))
 STORE = ResultsStore(RESULTS_REPO, _TOKEN, min_params=MIN_PARAMS)
 
 SUGGESTED = [
-    '56m/Dumb-1.2-RC1',
-    'allura-org/Rambley-150M-RealBase',
-    'altslate/JugnuLM-110M-R2plus',
-    'appvoid/void.0',
-    'AtomixLabs/AtomixS2-5M-v1.0',
     'AxiomicLabs/GPT-S-1.4M',
     'AxiomicLabs/GPT-S2-5M',
     'AxiomicLabs/GPT-X2.5-135M',
     'BananaMind/BananaMind-2-Medium',
     'BananaMind/BananaMind-2-Micro',
     'BananaMind/BananaMind-2-Mini',
-    'BananaMind/BananaMind-2-MoE',
     'BananaMind/BananaMind-2-Nano',
     'BananaMind/BananaMind-2-Pro',
     'BananaMind/BananaMind-2.1-Pico-Preview',
     'BananaMind/BananaMind-2.1-Unified',
-    'bench-labs/cagliostro-v3',
-    'CNWPlayer/VegaLM1-42M-Base',
-    'CodeSoft/sorbet-v2-25m',
+    'DedeBckp/BackKiyo-10M',
     'DedeProGames/DynamicMind-Mini',
     'DedeProGames/DynamicMind-MoE',
     'DedeProGames/Kiyo-230M-Preview',
@@ -59,58 +53,14 @@ SUGGESTED = [
     'DedeProGames/LowOnMind-1M',
     'DedeProGames/LowOnMind-300k',
     'DedeProGames/LowOnMind-5M',
-    'DedeProGames/LowOnMind-8M',
-    'DedeProGames/NanoDex-1M',
-    'EleutherAI/pythia-160m',
-    'EleutherAI/pythia-70m',
-    'finnianx/Gros-Michel-90m-Base-v2',
-    'FlameF0X/TinyMoE-100m-2x8-retrained',
-    'fromziro/ZeroS-Qana-5M',
-    'fromziro/ZeroS-v0.1-150M',
-    'FWKV/Myosotis-1-base',
-    'GODELEV/Rose-1.5-Medium',
-    'GODELEV/Rose-Mini',
-    'Harley-ml/Dillionv2-1.3M',
-    'HuggingFaceTB/SmolLM2-135M',
-    'IvmeLabs/Ivme-Conversate-v3-Base',
-    'jhu-clsp/ettin-decoder-150m',
-    'jhu-clsp/ettin-decoder-17m',
-    'jhu-clsp/ettin-decoder-32m',
-    'jhu-clsp/ettin-decoder-68m',
-    'joelhenwang/OdinNext-138M-Base',
-    'LH-Tech-AI/Spark-5M-Base-v4',
-    'MaliosDark/Nexus-Erebus-135M',
-    'MaliosDark/Nexus-Erebus-3M',
-    'MaliosDark/Nexus-Erebus-50M',
-    'MihaiPopa-1/CinnabarLM-1.4M-Base',
-    'MinimaLabs/KeyLM-75M',
-    'MinimaLabs/min-spark-1.1',
     'openai-community/gpt2',
-    'opencerebral/Boris-1.3-125M',
-    'opencerebral/Boris-1.3-75M',
-    'opencerebral/littlerock-1M',
-    'qikp/kite-7-15m-base',
-    'roneneldan/TinyStories-33M',
-    'SlayerLab/pollock-mini-lm-125m',
-    'StentorLabs/Stentor3-20M',
-    'StentorLabs/Stentor3-50M',
     'SupraLabs/Supra2-100M-Base',
     'SupraLabs/Supra2-Medium-Base',
-    'SupraLabs/SupraGDN-5M',
     'SupraLabs/SupraNeo-4M',
-    'SurjoLabs/Ember-2',
-    'SurjoLabs/Flare',
-    'SurjoLabs/Surjo-50m',
-    'sz14/cRia-LM-75M',
-    'TobiasLogic/Museko-125M',
-    'UniversalComputingResearch/Atom2.7m',
-    'User01110/CMA-8M',
-    'veyra-ai/Veyra2-Blueberry-10M-Base',
-    'veyra-ai/Veyra2-Blueberry-5M-Base',
 ]
-DEFAULT_MODELS = ["DedeProGames/Kiyo-65M", "BananaMind/BananaMind-2-Medium", "SurjoLabs/Surjo-50m", "StentorLabs/Stentor3-50M"]
+DEFAULT_MODELS = ["DedeProGames/Kiyo-65M", "BananaMind/BananaMind-2-Medium", "DedeProGames/DynamicMind-MoE", "SupraLabs/Supra2-Medium-Base"]
 # Ranked: the arena picks the players at random from the suggested models (nobody chooses who plays)
-POOL = Pool(SUGGESTED, MAX_PARAM_GAP, MAX_MODELS)
+POOL = Pool(SUGGESTED, MAX_PARAM_GAP, MAX_MODELS, LARGE_FROM)
 RANKED = RankedRunner(POOL, STORE)
 PROTOCOL_CHOICES = [("Guided: the rules are in the prompt", "guided"), ("Blind: no rules, only pre-training knowledge", "blind")]
 BASELINE_CHOICES = [(label, key) for key, label in BASELINES.items()]
@@ -336,7 +286,8 @@ Because the value is a difference, a model's general bias towards "good" or "bad
 - **Match** tab (friendly): pick any 2+ players (up to {MAX_MODELS} language models of any size, plus optional baselines)
   and the seed. Nothing is recorded.
 - **Ranked** tab: press Play and the arena picks up to {MAX_MODELS} models at random from its pool of {len(POOL.ids)} models,
-  all within {fmt_params(MAX_PARAM_GAP)} parameters of each other, with a random seed. Nobody chooses who plays, so Elo can't be farmed.
+  all within {fmt_params(MAX_PARAM_GAP)} parameters of each other (models with {short_params(LARGE_FROM)}+ parameters can all play each other),
+  with a random seed. Nobody chooses who plays, so Elo can't be farmed.
   Models with fewer ranked games are more likely to be picked, so every model gets played. The match runs on the server and counts even if
   you close the page; only one ranked match runs at a time, and pressing Play while one is running lets you watch it.
 - Elo: K=32, multiplayer (every pair of players counts as a game, scaled by 1/(N−1)).
@@ -358,7 +309,8 @@ Results and every match (seed, commit SHA of each model, scores) are published i
 """
 
 RANKED_INTRO = f"""<p class="ah-note">Press <b>Play</b> and the arena picks up to {MAX_MODELS} language models <b>at random</b>
-from its pool of {len(POOL.ids)} models, all within ±{short_params(MAX_PARAM_GAP)} parameters of each other, with a random seed.
+from its pool of {len(POOL.ids)} models, all within ±{short_params(MAX_PARAM_GAP)} parameters of each other
+(models with {short_params(LARGE_FROM)}+ parameters can all play each other), with a random seed.
 The result updates the public Elo leaderboard. The match runs on the server: it finishes and counts even if you close the page.
 Only one ranked match runs at a time; if one is already running, you watch it.</p>"""