| { |
| "chart": "multilingual", |
| "metric": "per-language MASSIVE intent accuracy, argmax over 20 scored options, 100 rows/language", |
| "sources": { |
| "v3": "runs/macjev/received/20260924-0313/evals/main/test/massive51.jsonl", |
| "laya_multilingual": "runs/macjev/laya_baselines/multilingual/massive51.jsonl", |
| "chart_data_entry": "chart_data.json#massive51.per_language (re-verified against raw files)" |
| }, |
| "protocol": "14 trained + 37 locales held out of MASSIVE training for v3; held-out for all Laya checkpoints. Laya numbers: official checkpoints re-run by us on identical rows, shipped temperatures. Wilson 95% CI per language (n=100)", |
| "chance": 0.05, |
| "three_x_chance": 0.15000000000000002, |
| "summary": { |
| "languages": 51, |
| "v3_beats_laya_multilingual": 51, |
| "v3_beats_best_laya_checkpoint": 51, |
| "v3_beats_laya_multilingual_never_trained": 37, |
| "v3_above_3x_chance": 51, |
| "laya_multilingual_above_3x_chance": 48, |
| "macro_v3": 0.7174509803921569, |
| "macro_laya_multilingual": 0.4007843137254902, |
| "macro_v3_never_trained37": 0.6551351351351351, |
| "macro_laya_multilingual_never_trained37": 0.36108108108108106, |
| "min_margin_vs_laya_multilingual": 0.11 |
| }, |
| "rows_plotted_sorted": [ |
| { |
| "lang": "zh-CN", |
| "name": "Chinese (CN)", |
| "trained": true, |
| "v3": 0.95, |
| "laya_multilingual": 0.65, |
| "best_laya": 0.65, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ja", |
| "name": "Japanese", |
| "trained": true, |
| "v3": 0.95, |
| "laya_multilingual": 0.64, |
| "best_laya": 0.64, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "es", |
| "name": "Spanish", |
| "trained": true, |
| "v3": 0.95, |
| "laya_multilingual": 0.58, |
| "best_laya": 0.58, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ru", |
| "name": "Russian", |
| "trained": true, |
| "v3": 0.94, |
| "laya_multilingual": 0.57, |
| "best_laya": 0.57, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "en", |
| "name": "English", |
| "trained": true, |
| "v3": 0.93, |
| "laya_multilingual": 0.71, |
| "best_laya": 0.82, |
| "best_laya_ckpt": "english" |
| }, |
| { |
| "lang": "zh-TW", |
| "name": "Chinese (TW)", |
| "trained": false, |
| "v3": 0.93, |
| "laya_multilingual": 0.61, |
| "best_laya": 0.61, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "de", |
| "name": "German", |
| "trained": true, |
| "v3": 0.93, |
| "laya_multilingual": 0.5, |
| "best_laya": 0.5, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "fr", |
| "name": "French", |
| "trained": true, |
| "v3": 0.9, |
| "laya_multilingual": 0.6, |
| "best_laya": 0.6, |
| "best_laya_ckpt": "typed" |
| }, |
| { |
| "lang": "it", |
| "name": "Italian", |
| "trained": false, |
| "v3": 0.89, |
| "laya_multilingual": 0.52, |
| "best_laya": 0.52, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "pl", |
| "name": "Polish", |
| "trained": false, |
| "v3": 0.89, |
| "laya_multilingual": 0.5, |
| "best_laya": 0.5, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ko", |
| "name": "Korean", |
| "trained": true, |
| "v3": 0.89, |
| "laya_multilingual": 0.47, |
| "best_laya": 0.47, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "pt", |
| "name": "Portuguese", |
| "trained": true, |
| "v3": 0.88, |
| "laya_multilingual": 0.5, |
| "best_laya": 0.5, |
| "best_laya_ckpt": "typed" |
| }, |
| { |
| "lang": "hi", |
| "name": "Hindi", |
| "trained": true, |
| "v3": 0.87, |
| "laya_multilingual": 0.46, |
| "best_laya": 0.46, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "sv", |
| "name": "Swedish", |
| "trained": false, |
| "v3": 0.85, |
| "laya_multilingual": 0.49, |
| "best_laya": 0.49, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ar", |
| "name": "Arabic", |
| "trained": true, |
| "v3": 0.85, |
| "laya_multilingual": 0.46, |
| "best_laya": 0.46, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "vi", |
| "name": "Vietnamese", |
| "trained": false, |
| "v3": 0.85, |
| "laya_multilingual": 0.34, |
| "best_laya": 0.34, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "fa", |
| "name": "Persian", |
| "trained": false, |
| "v3": 0.84, |
| "laya_multilingual": 0.51, |
| "best_laya": 0.51, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "id", |
| "name": "Indonesian", |
| "trained": false, |
| "v3": 0.84, |
| "laya_multilingual": 0.51, |
| "best_laya": 0.51, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "tr", |
| "name": "Turkish", |
| "trained": true, |
| "v3": 0.83, |
| "laya_multilingual": 0.4, |
| "best_laya": 0.4, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "nl", |
| "name": "Dutch", |
| "trained": false, |
| "v3": 0.82, |
| "laya_multilingual": 0.47, |
| "best_laya": 0.47, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "nb", |
| "name": "Norwegian", |
| "trained": false, |
| "v3": 0.81, |
| "laya_multilingual": 0.56, |
| "best_laya": 0.56, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "da", |
| "name": "Danish", |
| "trained": false, |
| "v3": 0.81, |
| "laya_multilingual": 0.52, |
| "best_laya": 0.52, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ro", |
| "name": "Romanian", |
| "trained": false, |
| "v3": 0.77, |
| "laya_multilingual": 0.37, |
| "best_laya": 0.37, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "af", |
| "name": "Afrikaans", |
| "trained": false, |
| "v3": 0.76, |
| "laya_multilingual": 0.35, |
| "best_laya": 0.35, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ta", |
| "name": "Tamil", |
| "trained": true, |
| "v3": 0.74, |
| "laya_multilingual": 0.31, |
| "best_laya": 0.31, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "sw", |
| "name": "Swahili", |
| "trained": true, |
| "v3": 0.74, |
| "laya_multilingual": 0.23, |
| "best_laya": 0.23, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ms", |
| "name": "Malay", |
| "trained": false, |
| "v3": 0.73, |
| "laya_multilingual": 0.41, |
| "best_laya": 0.41, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "el", |
| "name": "Greek", |
| "trained": false, |
| "v3": 0.72, |
| "laya_multilingual": 0.44, |
| "best_laya": 0.44, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "he", |
| "name": "Hebrew", |
| "trained": false, |
| "v3": 0.72, |
| "laya_multilingual": 0.37, |
| "best_laya": 0.37, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ur", |
| "name": "Urdu", |
| "trained": false, |
| "v3": 0.7, |
| "laya_multilingual": 0.42, |
| "best_laya": 0.42, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "az", |
| "name": "Azerbaijani", |
| "trained": false, |
| "v3": 0.66, |
| "laya_multilingual": 0.36, |
| "best_laya": 0.36, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "bn", |
| "name": "Bengali", |
| "trained": false, |
| "v3": 0.65, |
| "laya_multilingual": 0.45, |
| "best_laya": 0.45, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "sl", |
| "name": "Slovenian", |
| "trained": false, |
| "v3": 0.65, |
| "laya_multilingual": 0.37, |
| "best_laya": 0.37, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "th", |
| "name": "Thai", |
| "trained": false, |
| "v3": 0.64, |
| "laya_multilingual": 0.48, |
| "best_laya": 0.48, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "hu", |
| "name": "Hungarian", |
| "trained": false, |
| "v3": 0.63, |
| "laya_multilingual": 0.36, |
| "best_laya": 0.36, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "te", |
| "name": "Telugu", |
| "trained": false, |
| "v3": 0.62, |
| "laya_multilingual": 0.22, |
| "best_laya": 0.22, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "kn", |
| "name": "Kannada", |
| "trained": false, |
| "v3": 0.61, |
| "laya_multilingual": 0.3, |
| "best_laya": 0.3, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "fi", |
| "name": "Finnish", |
| "trained": false, |
| "v3": 0.6, |
| "laya_multilingual": 0.34, |
| "best_laya": 0.34, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "jv", |
| "name": "Javanese", |
| "trained": false, |
| "v3": 0.6, |
| "laya_multilingual": 0.3, |
| "best_laya": 0.3, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "hy", |
| "name": "Armenian", |
| "trained": false, |
| "v3": 0.59, |
| "laya_multilingual": 0.25, |
| "best_laya": 0.25, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ka", |
| "name": "Georgian", |
| "trained": false, |
| "v3": 0.56, |
| "laya_multilingual": 0.15, |
| "best_laya": 0.15, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "tl", |
| "name": "Tagalog", |
| "trained": false, |
| "v3": 0.54, |
| "laya_multilingual": 0.35, |
| "best_laya": 0.35, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "km", |
| "name": "Khmer", |
| "trained": false, |
| "v3": 0.54, |
| "laya_multilingual": 0.2, |
| "best_laya": 0.2, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "sq", |
| "name": "Albanian", |
| "trained": false, |
| "v3": 0.52, |
| "laya_multilingual": 0.3, |
| "best_laya": 0.3, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "ml", |
| "name": "Malayalam", |
| "trained": false, |
| "v3": 0.52, |
| "laya_multilingual": 0.28, |
| "best_laya": 0.28, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "lv", |
| "name": "Latvian", |
| "trained": false, |
| "v3": 0.51, |
| "laya_multilingual": 0.31, |
| "best_laya": 0.31, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "is", |
| "name": "Icelandic", |
| "trained": false, |
| "v3": 0.5, |
| "laya_multilingual": 0.35, |
| "best_laya": 0.35, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "my", |
| "name": "Burmese", |
| "trained": false, |
| "v3": 0.46, |
| "laya_multilingual": 0.16, |
| "best_laya": 0.16, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "cy", |
| "name": "Welsh", |
| "trained": false, |
| "v3": 0.37, |
| "laya_multilingual": 0.13, |
| "best_laya": 0.17, |
| "best_laya_ckpt": "typed" |
| }, |
| { |
| "lang": "mn", |
| "name": "Mongolian", |
| "trained": false, |
| "v3": 0.28, |
| "laya_multilingual": 0.16, |
| "best_laya": 0.16, |
| "best_laya_ckpt": "multilingual" |
| }, |
| { |
| "lang": "am", |
| "name": "Amharic", |
| "trained": false, |
| "v3": 0.26, |
| "laya_multilingual": 0.15, |
| "best_laya": 0.16, |
| "best_laya_ckpt": "typed" |
| } |
| ] |
| } |
