const validPercent = n => Number.isFinite(n) && n >= 0 && n <= 100; // Fixed Elo reference, independent of which models are selected or added. // This is an index normalization, not measured benchmark accuracy. const eloToScore = elo => Number.isFinite(elo) ? 100 / (1 + 10 ** ((1000 - elo) / 400)) : null; const benchmarkValue = (m, key) => key === 'base' ? eloToScore(m.base?.overall) : m[key]; function indexScore(model) { const values = BENCHMARKS.map(b => benchmarkValue(model,b.key)); if (!values.every(validPercent)) return null; return Math.max(1, values.reduce((sum,v,i) => sum + v * BENCHMARKS[i].weight,0)); } function sizeRadius(params) { // Thresholds are inclusive; the final ±20M rule continues above 125M. for (const [min,radius] of [[125e6,20e6],[100e6,15e6],[50e6,7.5e6],[25e6,5e6],[10e6,2e6],[5e6,1e6],[1e6,1e5]]) { if (params >= min) return radius; } return null; } function frontierInfo(model, models) { const radius = sizeRadius(model.params), score = indexScore(model); if (radius === null || score === null || model.excludedSizeComparison) return {frontier:false, peers:[],radius}; const peers = models.filter(m => !m.excludedSizeComparison && indexScore(m) !== null && Math.abs(m.params-model.params) <= radius); return {frontier:peers.every(m => indexScore(m) <= score), peers, radius}; } function profile(model) { const b = Object.fromEntries(Object.entries(model.base?.elo || {}).map(([key,value]) => [key,eloToScore(value)])); const mix = (a,c,w) => validPercent(a) && validPercent(c) ? a*(1-w)+c*w : null; return [ ['Language', b.language, 'Base Bench · normalized language Elo'], ['Commonsense', mix(b.commonsense,model.hellaswag,0.5), '50% Base Bench commonsense + 50% HellaSwag'], ['World knowledge', b.knowledge, 'Base Bench · normalized world knowledge Elo'], ['Knowledge', model.arc, 'ARC Easy accuracy'], ['Quantitative', mix(b.quantitative,model.arithmark3,0.1), '90% Base Bench quantitative + 10% Arithmark 3'], ['Logic', b.logic, 'Base Bench · normalized logical reasoning Elo'], ['Code', b.code, 'Base Bench · normalized code completion Elo'], ['Physical', model.piqa, 'PIQA accuracy'], ]; }