opensysone / source /results /20260917-fleet-progress /spark-b-refinement-initial-validation-selection.json
| { | |
| "metric": "crossfit_temperature_nll_v1", | |
| "score": 0.18857624356360508, | |
| "raw_macro_nll": 0.30707557289052434, | |
| "accuracy": 0.935546875, | |
| "folds": [ | |
| { | |
| "fold": 0, | |
| "temperature_index": 60, | |
| "temperature": 2.39883291901949, | |
| "training_macro_nll": 0.17825165720392558, | |
| "heldout_ids": [ | |
| "arc:validation:ARC-Challenge:122", | |
| "arc:validation:ARC-Challenge:126", | |
| "arc:validation:ARC-Challenge:175", | |
| "arc:validation:ARC-Challenge:187", | |
| "arc:validation:ARC-Challenge:189", | |
| "arc:validation:ARC-Challenge:192", | |
| "arc:validation:ARC-Challenge:208", | |
| "arc:validation:ARC-Challenge:238", | |
| "arc:validation:ARC-Challenge:290", | |
| "arc:validation:ARC-Challenge:293", | |
| "arc:validation:ARC-Challenge:35", | |
| "arc:validation:ARC-Challenge:43", | |
| "arc:validation:ARC-Challenge:58", | |
| "arc:validation:ARC-Challenge:7", | |
| "arc:validation:ARC-Easy:160", | |
| "arc:validation:ARC-Easy:201", | |
| "arc:validation:ARC-Easy:229", | |
| "arc:validation:ARC-Easy:238", | |
| "arc:validation:ARC-Easy:253", | |
| "arc:validation:ARC-Easy:259", | |
| "arc:validation:ARC-Easy:272", | |
| "arc:validation:ARC-Easy:298", | |
| "arc:validation:ARC-Easy:329", | |
| "arc:validation:ARC-Easy:331", | |
| "arc:validation:ARC-Easy:337", | |
| "arc:validation:ARC-Easy:362", | |
| "arc:validation:ARC-Easy:443", | |
| "arc:validation:ARC-Easy:500", | |
| "arc:validation:ARC-Easy:545", | |
| "arc:validation:ARC-Easy:68", | |
| "arc:validation:ARC-Easy:7", | |
| "arc:validation:ARC-Easy:74", | |
| "banking:train:1053", | |
| "banking:train:1594", | |
| "banking:train:1747", | |
| "banking:train:202", | |
| "banking:train:2377", | |
| "banking:train:2528", | |
| "banking:train:2546", | |
| "banking:train:2715", | |
| "banking:train:3220", | |
| "banking:train:3499", | |
| "banking:train:3806", | |
| "banking:train:3838", | |
| "banking:train:3993", | |
| "banking:train:4220", | |
| "banking:train:5", | |
| "banking:train:5192", | |
| "banking:train:5394", | |
| "banking:train:5846", | |
| "banking:train:5863", | |
| "banking:train:6032", | |
| "banking:train:6508", | |
| "banking:train:7579", | |
| "banking:train:7600", | |
| "banking:train:7987", | |
| "banking:train:8155", | |
| "banking:train:850", | |
| "banking:train:8626", | |
| "banking:train:8929", | |
| "banking:train:9355", | |
| "banking:train:9812", | |
| "banking:train:987", | |
| "banking:train:9972", | |
| "boolq:validation:103", | |
| "boolq:validation:1042", | |
| "boolq:validation:1059", | |
| "boolq:validation:1541", | |
| "boolq:validation:1592", | |
| "boolq:validation:1927", | |
| "boolq:validation:2014", | |
| "boolq:validation:2023", | |
| "boolq:validation:2098", | |
| "boolq:validation:2139", | |
| "boolq:validation:2439", | |
| "boolq:validation:2485", | |
| "boolq:validation:2607", | |
| "boolq:validation:2611", | |
| "boolq:validation:2720", | |
| "boolq:validation:2751", | |
| "boolq:validation:282", | |
| "boolq:validation:2843", | |
| "boolq:validation:2953", | |
| "boolq:validation:301", | |
| "boolq:validation:3089", | |
| "boolq:validation:313", | |
| "boolq:validation:3201", | |
| "boolq:validation:322", | |
| "boolq:validation:3231", | |
| "boolq:validation:39", | |
| "boolq:validation:461", | |
| "boolq:validation:558", | |
| "boolq:validation:589", | |
| "boolq:validation:731", | |
| "boolq:validation:883", | |
| "boolq:validation:915", | |
| "snli:validation:1156", | |
| "snli:validation:1330", | |
| "snli:validation:1559", | |
| "snli:validation:1735", | |
| "snli:validation:1876", | |
| "snli:validation:2251", | |
| "snli:validation:2257", | |
| "snli:validation:2377", | |
| "snli:validation:2881", | |
| "snli:validation:291", | |
| "snli:validation:300", | |
| "snli:validation:3049", | |
| "snli:validation:3496", | |
| "snli:validation:3871", | |
| "snli:validation:4414", | |
| "snli:validation:5140", | |
| "snli:validation:5440", | |
| "snli:validation:5806", | |
| "snli:validation:6442", | |
| "snli:validation:6772", | |
| "snli:validation:6949", | |
| "snli:validation:7228", | |
| "snli:validation:7465", | |
| "snli:validation:7510", | |
| "snli:validation:7726", | |
| "snli:validation:8095", | |
| "snli:validation:8143", | |
| "snli:validation:8623", | |
| "snli:validation:8962", | |
| "snli:validation:9451", | |
| "snli:validation:9574", | |
| "snli:validation:9655" | |
| ], | |
| "heldout_group_count": 128, | |
| "training_group_count": 384 | |
| }, | |
| { | |
| "fold": 1, | |
| "temperature_index": 61, | |
| "temperature": 2.5292979964461435, | |
| "training_macro_nll": 0.17419859454279013, | |
| "heldout_ids": [ | |
| "arc:validation:ARC-Challenge:101", | |
| "arc:validation:ARC-Challenge:129", | |
| "arc:validation:ARC-Challenge:148", | |
| "arc:validation:ARC-Challenge:156", | |
| "arc:validation:ARC-Challenge:167", | |
| "arc:validation:ARC-Challenge:249", | |
| "arc:validation:ARC-Challenge:263", | |
| "arc:validation:ARC-Challenge:266", | |
| "arc:validation:ARC-Challenge:267", | |
| "arc:validation:ARC-Challenge:28", | |
| "arc:validation:ARC-Challenge:281", | |
| "arc:validation:ARC-Challenge:57", | |
| "arc:validation:ARC-Challenge:63", | |
| "arc:validation:ARC-Challenge:64", | |
| "arc:validation:ARC-Challenge:86", | |
| "arc:validation:ARC-Challenge:99", | |
| "arc:validation:ARC-Easy:106", | |
| "arc:validation:ARC-Easy:151", | |
| "arc:validation:ARC-Easy:157", | |
| "arc:validation:ARC-Easy:166", | |
| "arc:validation:ARC-Easy:202", | |
| "arc:validation:ARC-Easy:219", | |
| "arc:validation:ARC-Easy:224", | |
| "arc:validation:ARC-Easy:322", | |
| "arc:validation:ARC-Easy:344", | |
| "arc:validation:ARC-Easy:382", | |
| "arc:validation:ARC-Easy:409", | |
| "arc:validation:ARC-Easy:455", | |
| "arc:validation:ARC-Easy:50", | |
| "arc:validation:ARC-Easy:538", | |
| "arc:validation:ARC-Easy:544", | |
| "arc:validation:ARC-Easy:552", | |
| "banking:train:1050", | |
| "banking:train:1131", | |
| "banking:train:1190", | |
| "banking:train:124", | |
| "banking:train:137", | |
| "banking:train:1493", | |
| "banking:train:1498", | |
| "banking:train:1624", | |
| "banking:train:2614", | |
| "banking:train:2739", | |
| "banking:train:3055", | |
| "banking:train:306", | |
| "banking:train:3312", | |
| "banking:train:3724", | |
| "banking:train:4070", | |
| "banking:train:476", | |
| "banking:train:4882", | |
| "banking:train:5075", | |
| "banking:train:5332", | |
| "banking:train:5445", | |
| "banking:train:6096", | |
| "banking:train:746", | |
| "banking:train:7828", | |
| "banking:train:8176", | |
| "banking:train:8315", | |
| "banking:train:8653", | |
| "banking:train:9019", | |
| "banking:train:9370", | |
| "banking:train:9454", | |
| "banking:train:9595", | |
| "banking:train:9862", | |
| "banking:train:9963", | |
| "boolq:validation:1189", | |
| "boolq:validation:1320", | |
| "boolq:validation:1326", | |
| "boolq:validation:1381", | |
| "boolq:validation:1410", | |
| "boolq:validation:149", | |
| "boolq:validation:1513", | |
| "boolq:validation:1732", | |
| "boolq:validation:1739", | |
| "boolq:validation:1741", | |
| "boolq:validation:1814", | |
| "boolq:validation:1848", | |
| "boolq:validation:1940", | |
| "boolq:validation:2077", | |
| "boolq:validation:2091", | |
| "boolq:validation:2148", | |
| "boolq:validation:2215", | |
| "boolq:validation:2251", | |
| "boolq:validation:2283", | |
| "boolq:validation:2321", | |
| "boolq:validation:2398", | |
| "boolq:validation:253", | |
| "boolq:validation:2575", | |
| "boolq:validation:2907", | |
| "boolq:validation:2909", | |
| "boolq:validation:2978", | |
| "boolq:validation:359", | |
| "boolq:validation:394", | |
| "boolq:validation:436", | |
| "boolq:validation:579", | |
| "boolq:validation:807", | |
| "boolq:validation:949", | |
| "snli:validation:2389", | |
| "snli:validation:2485", | |
| "snli:validation:255", | |
| "snli:validation:3169", | |
| "snli:validation:333", | |
| "snli:validation:3700", | |
| "snli:validation:3709", | |
| "snli:validation:4093", | |
| "snli:validation:420", | |
| "snli:validation:4492", | |
| "snli:validation:4516", | |
| "snli:validation:4648", | |
| "snli:validation:4888", | |
| "snli:validation:5185", | |
| "snli:validation:5332", | |
| "snli:validation:5947", | |
| "snli:validation:6124", | |
| "snli:validation:660", | |
| "snli:validation:6619", | |
| "snli:validation:6898", | |
| "snli:validation:7387", | |
| "snli:validation:7492", | |
| "snli:validation:7544", | |
| "snli:validation:7996", | |
| "snli:validation:8074", | |
| "snli:validation:8554", | |
| "snli:validation:8971", | |
| "snli:validation:9163", | |
| "snli:validation:9277", | |
| "snli:validation:9490", | |
| "snli:validation:9595", | |
| "snli:validation:9688" | |
| ], | |
| "heldout_group_count": 128, | |
| "training_group_count": 384 | |
| }, | |
| { | |
| "fold": 2, | |
| "temperature_index": 62, | |
| "temperature": 2.666858664521479, | |
| "training_macro_nll": 0.18663432029335247, | |
| "heldout_ids": [ | |
| "arc:validation:ARC-Challenge:120", | |
| "arc:validation:ARC-Challenge:130", | |
| "arc:validation:ARC-Challenge:194", | |
| "arc:validation:ARC-Challenge:218", | |
| "arc:validation:ARC-Challenge:235", | |
| "arc:validation:ARC-Challenge:272", | |
| "arc:validation:ARC-Challenge:277", | |
| "arc:validation:ARC-Challenge:48", | |
| "arc:validation:ARC-Challenge:88", | |
| "arc:validation:ARC-Easy:115", | |
| "arc:validation:ARC-Easy:12", | |
| "arc:validation:ARC-Easy:131", | |
| "arc:validation:ARC-Easy:145", | |
| "arc:validation:ARC-Easy:179", | |
| "arc:validation:ARC-Easy:18", | |
| "arc:validation:ARC-Easy:186", | |
| "arc:validation:ARC-Easy:191", | |
| "arc:validation:ARC-Easy:215", | |
| "arc:validation:ARC-Easy:283", | |
| "arc:validation:ARC-Easy:314", | |
| "arc:validation:ARC-Easy:315", | |
| "arc:validation:ARC-Easy:378", | |
| "arc:validation:ARC-Easy:384", | |
| "arc:validation:ARC-Easy:403", | |
| "arc:validation:ARC-Easy:432", | |
| "arc:validation:ARC-Easy:439", | |
| "arc:validation:ARC-Easy:448", | |
| "arc:validation:ARC-Easy:461", | |
| "arc:validation:ARC-Easy:469", | |
| "arc:validation:ARC-Easy:470", | |
| "arc:validation:ARC-Easy:504", | |
| "arc:validation:ARC-Easy:561", | |
| "banking:train:1142", | |
| "banking:train:148", | |
| "banking:train:1633", | |
| "banking:train:1927", | |
| "banking:train:2186", | |
| "banking:train:2251", | |
| "banking:train:23", | |
| "banking:train:3048", | |
| "banking:train:3665", | |
| "banking:train:3952", | |
| "banking:train:3999", | |
| "banking:train:4364", | |
| "banking:train:4619", | |
| "banking:train:5049", | |
| "banking:train:5335", | |
| "banking:train:549", | |
| "banking:train:5957", | |
| "banking:train:5985", | |
| "banking:train:6132", | |
| "banking:train:6240", | |
| "banking:train:6488", | |
| "banking:train:7065", | |
| "banking:train:7749", | |
| "banking:train:8470", | |
| "banking:train:8761", | |
| "banking:train:895", | |
| "banking:train:9256", | |
| "banking:train:9406", | |
| "banking:train:9450", | |
| "banking:train:9467", | |
| "banking:train:9588", | |
| "banking:train:9716", | |
| "boolq:validation:1130", | |
| "boolq:validation:1167", | |
| "boolq:validation:124", | |
| "boolq:validation:1255", | |
| "boolq:validation:1256", | |
| "boolq:validation:1373", | |
| "boolq:validation:15", | |
| "boolq:validation:155", | |
| "boolq:validation:1568", | |
| "boolq:validation:16", | |
| "boolq:validation:1654", | |
| "boolq:validation:1702", | |
| "boolq:validation:1792", | |
| "boolq:validation:1818", | |
| "boolq:validation:210", | |
| "boolq:validation:2247", | |
| "boolq:validation:2259", | |
| "boolq:validation:2362", | |
| "boolq:validation:2443", | |
| "boolq:validation:2524", | |
| "boolq:validation:2553", | |
| "boolq:validation:2623", | |
| "boolq:validation:2660", | |
| "boolq:validation:2669", | |
| "boolq:validation:2679", | |
| "boolq:validation:2832", | |
| "boolq:validation:357", | |
| "boolq:validation:453", | |
| "boolq:validation:49", | |
| "boolq:validation:808", | |
| "boolq:validation:90", | |
| "boolq:validation:910", | |
| "snli:validation:1612", | |
| "snli:validation:168", | |
| "snli:validation:1693", | |
| "snli:validation:2134", | |
| "snli:validation:2248", | |
| "snli:validation:2422", | |
| "snli:validation:2587", | |
| "snli:validation:3007", | |
| "snli:validation:3097", | |
| "snli:validation:3772", | |
| "snli:validation:4012", | |
| "snli:validation:4168", | |
| "snli:validation:4537", | |
| "snli:validation:4681", | |
| "snli:validation:5050", | |
| "snli:validation:5632", | |
| "snli:validation:5761", | |
| "snli:validation:6055", | |
| "snli:validation:6283", | |
| "snli:validation:6715", | |
| "snli:validation:7117", | |
| "snli:validation:7192", | |
| "snli:validation:7507", | |
| "snli:validation:7594", | |
| "snli:validation:7924", | |
| "snli:validation:7954", | |
| "snli:validation:8035", | |
| "snli:validation:8110", | |
| "snli:validation:8995", | |
| "snli:validation:949", | |
| "snli:validation:9523", | |
| "snli:validation:9589" | |
| ], | |
| "heldout_group_count": 128, | |
| "training_group_count": 384 | |
| }, | |
| { | |
| "fold": 3, | |
| "temperature_index": 62, | |
| "temperature": 2.666858664521479, | |
| "training_macro_nll": 0.2066205595106239, | |
| "heldout_ids": [ | |
| "arc:validation:ARC-Challenge:117", | |
| "arc:validation:ARC-Challenge:135", | |
| "arc:validation:ARC-Challenge:139", | |
| "arc:validation:ARC-Challenge:154", | |
| "arc:validation:ARC-Challenge:174", | |
| "arc:validation:ARC-Challenge:199", | |
| "arc:validation:ARC-Challenge:287", | |
| "arc:validation:ARC-Challenge:295", | |
| "arc:validation:ARC-Challenge:4", | |
| "arc:validation:ARC-Challenge:50", | |
| "arc:validation:ARC-Challenge:51", | |
| "arc:validation:ARC-Challenge:61", | |
| "arc:validation:ARC-Easy:107", | |
| "arc:validation:ARC-Easy:121", | |
| "arc:validation:ARC-Easy:124", | |
| "arc:validation:ARC-Easy:171", | |
| "arc:validation:ARC-Easy:235", | |
| "arc:validation:ARC-Easy:241", | |
| "arc:validation:ARC-Easy:279", | |
| "arc:validation:ARC-Easy:289", | |
| "arc:validation:ARC-Easy:37", | |
| "arc:validation:ARC-Easy:389", | |
| "arc:validation:ARC-Easy:394", | |
| "arc:validation:ARC-Easy:399", | |
| "arc:validation:ARC-Easy:423", | |
| "arc:validation:ARC-Easy:492", | |
| "arc:validation:ARC-Easy:499", | |
| "arc:validation:ARC-Easy:523", | |
| "arc:validation:ARC-Easy:533", | |
| "arc:validation:ARC-Easy:569", | |
| "arc:validation:ARC-Easy:69", | |
| "arc:validation:ARC-Easy:93", | |
| "banking:train:2049", | |
| "banking:train:2350", | |
| "banking:train:2497", | |
| "banking:train:2506", | |
| "banking:train:2720", | |
| "banking:train:2788", | |
| "banking:train:2967", | |
| "banking:train:3502", | |
| "banking:train:3726", | |
| "banking:train:4014", | |
| "banking:train:4735", | |
| "banking:train:4803", | |
| "banking:train:4917", | |
| "banking:train:5477", | |
| "banking:train:5851", | |
| "banking:train:6551", | |
| "banking:train:6839", | |
| "banking:train:6926", | |
| "banking:train:7809", | |
| "banking:train:8092", | |
| "banking:train:8186", | |
| "banking:train:8215", | |
| "banking:train:8667", | |
| "banking:train:8752", | |
| "banking:train:8763", | |
| "banking:train:8851", | |
| "banking:train:8917", | |
| "banking:train:9145", | |
| "banking:train:9279", | |
| "banking:train:9676", | |
| "banking:train:9837", | |
| "banking:train:9967", | |
| "boolq:validation:106", | |
| "boolq:validation:1065", | |
| "boolq:validation:1097", | |
| "boolq:validation:1336", | |
| "boolq:validation:1350", | |
| "boolq:validation:1383", | |
| "boolq:validation:1458", | |
| "boolq:validation:1641", | |
| "boolq:validation:1873", | |
| "boolq:validation:1930", | |
| "boolq:validation:1989", | |
| "boolq:validation:2103", | |
| "boolq:validation:234", | |
| "boolq:validation:2355", | |
| "boolq:validation:2384", | |
| "boolq:validation:2440", | |
| "boolq:validation:2444", | |
| "boolq:validation:2478", | |
| "boolq:validation:2872", | |
| "boolq:validation:2879", | |
| "boolq:validation:2924", | |
| "boolq:validation:2971", | |
| "boolq:validation:3164", | |
| "boolq:validation:4", | |
| "boolq:validation:415", | |
| "boolq:validation:419", | |
| "boolq:validation:428", | |
| "boolq:validation:512", | |
| "boolq:validation:563", | |
| "boolq:validation:643", | |
| "boolq:validation:696", | |
| "boolq:validation:874", | |
| "snli:validation:1102", | |
| "snli:validation:1240", | |
| "snli:validation:1261", | |
| "snli:validation:1738", | |
| "snli:validation:1978", | |
| "snli:validation:2563", | |
| "snli:validation:2659", | |
| "snli:validation:2767", | |
| "snli:validation:2836", | |
| "snli:validation:3232", | |
| "snli:validation:3655", | |
| "snli:validation:3913", | |
| "snli:validation:405", | |
| "snli:validation:4051", | |
| "snli:validation:4300", | |
| "snli:validation:4846", | |
| "snli:validation:498", | |
| "snli:validation:5224", | |
| "snli:validation:5575", | |
| "snli:validation:6598", | |
| "snli:validation:666", | |
| "snli:validation:6925", | |
| "snli:validation:6964", | |
| "snli:validation:7045", | |
| "snli:validation:7351", | |
| "snli:validation:7588", | |
| "snli:validation:7723", | |
| "snli:validation:7750", | |
| "snli:validation:7909", | |
| "snli:validation:8974", | |
| "snli:validation:9385", | |
| "snli:validation:9646" | |
| ], | |
| "heldout_group_count": 128, | |
| "training_group_count": 384 | |
| } | |
| ], | |
| "per_family": { | |
| "arc": { | |
| "count": 128, | |
| "score": 0.1484131277186121, | |
| "raw_macro_nll": 0.10749165241994015, | |
| "accuracy": 0.953125 | |
| }, | |
| "banking": { | |
| "count": 128, | |
| "score": 0.06336078268803602, | |
| "raw_macro_nll": 0.09331359169251714, | |
| "accuracy": 0.9765625 | |
| }, | |
| "boolq": { | |
| "count": 128, | |
| "score": 0.3118573585109014, | |
| "raw_macro_nll": 0.628974288628438, | |
| "accuracy": 0.890625 | |
| }, | |
| "snli": { | |
| "count": 128, | |
| "score": 0.23067370533687087, | |
| "raw_macro_nll": 0.3985227588212021, | |
| "accuracy": 0.921875 | |
| } | |
| }, | |
| "policy": { | |
| "id": "crossfit_temperature_nll_v1", | |
| "folds": 4, | |
| "seed": 431, | |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", | |
| "temperature_grid": { | |
| "count": 101, | |
| "log10_min": -1.0, | |
| "log10_max": 1.3, | |
| "arithmetic": "Python float64" | |
| }, | |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", | |
| "score": "macro-family mean of all held-out decision NLLs", | |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" | |
| } | |
| } | |