{ "checked_utc": "2026-09-17T02:38:10.902428+00:00", "campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "source_status": "", "pilot_exit_code": 0, "verification_exit_code": 0, "training_exit_code": null, "campaign_exit_code": null, "initial_512_exact": true, "initial_predictions_sha256": "b2a477b919ef21569299b4adf3b8c6d78806659e190dee622fbf393b16d79905", "first_updates": [ { "step": 9, "loss": 2.443780346084168e-06, "gradient_norm": 0.0001523124083178118, "seconds": 9.567195054994954, "decisions": 4, "branches": 14, "actual_branch_tokens": 1407, "padded_branch_tokens": 1407, "peak_cuda_allocated_bytes": 16488540672, "lr_factor": 0.09 }, { "step": 10, "loss": 4.83664812236384e-05, "gradient_norm": 0.0019908652175217867, "seconds": 8.618153823998, "decisions": 4, "branches": 12, "actual_branch_tokens": 1331, "padded_branch_tokens": 1331, "peak_cuda_allocated_bytes": 16488540672, "lr_factor": 0.1 }, { "step": 11, "loss": 2.7762499842787207, "gradient_norm": 16.152069091796875, "seconds": 7.425390743999742, "decisions": 4, "branches": 9, "actual_branch_tokens": 1361, "padded_branch_tokens": 1361, "peak_cuda_allocated_bytes": 16516955648, "lr_factor": 0.11 }, { "step": 12, "loss": 5.048298348242497e-05, "gradient_norm": 0.0022558781784027815, "seconds": 8.469508371999837, "decisions": 4, "branches": 11, "actual_branch_tokens": 1533, "padded_branch_tokens": 1533, "peak_cuda_allocated_bytes": 16582566912, "lr_factor": 0.12 } ], "latest_update": { "step": 12, "loss": 5.048298348242497e-05, "gradient_norm": 0.0022558781784027815, "seconds": 8.469508371999837, "decisions": 4, "branches": 11, "actual_branch_tokens": 1533, "padded_branch_tokens": 1533, "peak_cuda_allocated_bytes": 16582566912, "lr_factor": 0.12 }, "correctness_initial": { "branch_chunks_1_probability_max_abs": 0.0, "branch_chunks_2_probability_max_abs": 3.637978807091713e-09, "branch_chunks_4_probability_max_abs": 2.153683453798294e-09, "question_isolation_probability_max_abs": 0.0, "candidate_permutation_probability_max_abs": 0.0, "repeat_probability_max_abs": 0.0, "tolerance_probability_abs": 0.0001 }, "initial_selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.18857624356360508, "raw_macro_nll": 0.30707557289052434, "accuracy": 0.935546875, "folds": [ { "fold": 0, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.17825165720392558, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 61, "temperature": 2.5292979964461435, "training_macro_nll": 0.17419859454279013, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.18663432029335247, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.2066205595106239, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.1484131277186121, "raw_macro_nll": 0.10749165241994015, "accuracy": 0.953125 }, "banking": { "count": 128, "score": 0.06336078268803602, "raw_macro_nll": 0.09331359169251714, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.3118573585109014, "raw_macro_nll": 0.628974288628438, "accuracy": 0.890625 }, "snli": { "count": 128, "score": 0.23067370533687087, "raw_macro_nll": 0.3985227588212021, "accuracy": 0.921875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "inherited_selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.18864030037724333, "raw_macro_nll": 0.30729043330681044, "accuracy": 0.935546875, "folds": [ { "fold": 0, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.17835820449058912, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 61, "temperature": 2.5292979964461435, "training_macro_nll": 0.17429139103918154, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.18674345787632526, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.20658826149875223, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.14837152412562352, "raw_macro_nll": 0.10747530600586384, "accuracy": 0.953125 }, "banking": { "count": 128, "score": 0.06337688481028368, "raw_macro_nll": 0.09334830709159572, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.3119620718470344, "raw_macro_nll": 0.6294207016878283, "accuracy": 0.890625 }, "snli": { "count": 128, "score": 0.23085072072603174, "raw_macro_nll": 0.3989174184419539, "accuracy": 0.921875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "plan": { "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "resume_sha256": "ece94b096b7fa37eec9e180a5f652da86a9f4f42d49f7f124850f046915237a8", "final_deadline": "2026-09-17T18:16:10+00:00", "eval_reserve_seconds": 7200, "evaluation_estimate": { "pilot_validation_decisions": 512, "pilot_validation_seconds": 317.40339064598083, "evaluation_decisions": 6640, "estimated_prediction_seconds": 4116.325222440064 }, "port": 18081, "training_deadline": "2026-09-17T16:00:00+00:00", "train_only": true, "selection_metric": "crossfit_temperature_nll_v1", "inference_max_tokens": 1024, "created_utc": "2026-09-17T02:31:37.778693+00:00", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" }, "state": { "supervisor_pid": 483974, "campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h", "stage": "training", "status": "running", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "child_pid": 484001, "child_command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/experiment.py", "train", "--output", "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training", "--resume", "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "--deadline", "2026-09-17T16:00:00+00:00", "--model", "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "--dataset", "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "--epochs", "3", "--rank", "8", "--alpha", "16.0", "--lr", "1e-05", "--head-lr", "1e-05", "--seed", "432", "--effective-batch", "4", "--branch-batch-size", "1", "--max-tokens", "512", "--save-steps", "250", "--save-seconds", "900", "--eval-steps", "500", "--validation-per-family", "128", "--patience", "8", "--schedule-steps", "5000", "--selection-metric", "crossfit_temperature_nll_v1", "--two-pass" ], "stage_started_utc": "2026-09-17T02:31:38.483925+00:00", "heartbeat_utc": "2026-09-17T02:38:08.559650+00:00" }, "processes": { "supervisor_pid": { "pid": 483974, "command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/scripts/launch_24h.py", "--campaign", "/home/andy/ai/opensysone/runs/20260917T023137Z-24h" ], "oom_score_adj": "0" }, "child_pid": { "pid": 484001, "command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/experiment.py", "train", "--output", "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training", "--resume", "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "--deadline", "2026-09-17T16:00:00+00:00", "--model", "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "--dataset", "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "--epochs", "3", "--rank", "8", "--alpha", "16.0", "--lr", "1e-05", "--head-lr", "1e-05", "--seed", "432", "--effective-batch", "4", "--branch-batch-size", "1", "--max-tokens", "512", "--save-steps", "250", "--save-seconds", "900", "--eval-steps", "500", "--validation-per-family", "128", "--patience", "8", "--schedule-steps", "5000", "--selection-metric", "crossfit_temperature_nll_v1", "--two-pass" ], "oom_score_adj": "0" } }, "resume_proof": { "checked_utc": "2026-09-17T02:32:09.829023+00:00", "campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h", "pilot": "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts", "step": 8, "exact_restoration": { "trainable_state": true, "optimizer": true, "random_state": true, "torch_rng": true, "cuda_rng": true, "model_provenance": true, "data_signature": true }, "config_preserved": { "rank": 8, "alpha": 16.0, "adapters": true, "max_tokens": 512, "branch_batch_size": 1, "seed": 432, "effective_batch": 4, "lr": 1e-05, "head_lr": 1e-05, "schedule_steps": 5000, "epochs": 3, "validation_per_family": 128, "two_pass": true, "selection_metric": "crossfit_temperature_nll_v1" }, "processes": { "supervisor_pid": { "pid": 483974, "command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/scripts/launch_24h.py", "--campaign", "/home/andy/ai/opensysone/runs/20260917T023137Z-24h" ], "oom_score_adj": "0" }, "child_pid": { "pid": 484001, "command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/experiment.py", "train", "--output", "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training", "--resume", "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "--deadline", "2026-09-17T16:00:00+00:00", "--model", "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "--dataset", "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "--epochs", "3", "--rank", "8", "--alpha", "16.0", "--lr", "1e-05", "--head-lr", "1e-05", "--seed", "432", "--effective-batch", "4", "--branch-batch-size", "1", "--max-tokens", "512", "--save-steps", "250", "--save-seconds", "900", "--eval-steps", "500", "--validation-per-family", "128", "--patience", "8", "--schedule-steps", "5000", "--selection-metric", "crossfit_temperature_nll_v1", "--two-pass" ], "oom_score_adj": "0" } }, "pilot_checkpoint_sha256": "ece94b096b7fa37eec9e180a5f652da86a9f4f42d49f7f124850f046915237a8", "pilot_step8_predictions_sha256": "b2a477b919ef21569299b4adf3b8c6d78806659e190dee622fbf393b16d79905", "campaign_initialization": { "kind": "resume", "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "parent_checkpoint_sha256": "ece94b096b7fa37eec9e180a5f652da86a9f4f42d49f7f124850f046915237a8", "restores_optimizer": true, "restores_rng": true, "parent_step": 8, "parent_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "parent_initialization": { "kind": "warm_start", "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260917T021254Z-refinement-parent/best.pt", "parent_checkpoint_sha256": "8956eb6c0cfbb02124aeefd99c3b418c55f55fdb9a64260350622d98dbba1aec", "restores_optimizer": false, "restores_rng": false, "parent_step": 2500, "parent_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" }, "selection_policy_change": { "from": "crossfit_temperature_nll_v1", "to": "crossfit_temperature_nll_v1", "optimizer_and_rng_unchanged": true } }, "optimizer_step_values": [ 8.0 ], "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" }, "scope": "Startup/validation proof only; no reserved calibration/test/holdout predictions accessed." }