{ "selected": { "name": "gx10-4b-expanded", "host": "local", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260917T075209Z-gx10-final-validation", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/training/best.pt", "checkpoint_sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca", "step": 0, "training_source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86", "model_provenance": { "model_id": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "license": "apache-2.0" }, "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917", "output": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts", "resume": null, "warm_start": "/home/andy/ai/opensysone/runs/20260917T070415Z-expanded-parent/best.pt", "allow_train_data_change": true, "steps": 8, "epochs": 3, "rank": 8, "alpha": 16.0, "head_only": false, "two_pass": true, "lr": 2e-05, "head_lr": 2e-05, "seed": 433, "effective_batch": 4, "branch_batch_size": 1, "max_tokens": 512, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00Z", "schedule_steps": 3500, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/evidence_0/best_validation_predictions.json", "prediction_sha256": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "metrics": { "macro_nll": 0.190872636672039, "accuracy": 0.947265625, "count": 512, "per_family_nll": { "arc": 0.1125077638524943, "banking": 0.05960886883339138, "boolq": 0.3498694938007437, "snli": 0.24150442020152654 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.1701497127614862, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.1701497127614862, "raw_macro_nll": 0.190872636672039, "accuracy": 0.947265625, "folds": [ { "fold": 0, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.1622086936723931, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15901522343009294, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15824280619091902, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.19176642571330008, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.13049231104529602, "raw_macro_nll": 0.1125077638524943, "accuracy": 0.96875 }, "banking": { "count": 128, "score": 0.05035731679828836, "raw_macro_nll": 0.05960886883339138, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.28226935283016463, "raw_macro_nll": 0.3498694938007437, "accuracy": 0.9296875 }, "snli": { "count": 128, "score": 0.21747987037219577, "raw_macro_nll": 0.24150442020152654, "accuracy": 0.9140625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/training/correctness_final.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts" ], "evidence_sha256": { "evidence_0/validation_step_000008_predictions.json": "8f0037152e67aebba05db28024bf02fc7fcb84986795efd464d171c2bc4dddcf", "evidence_0/initial_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "evidence_0/summary.json": "bc5e3ad91ecf3ef13ed3b82e408ebdd7110102a924f8b046e5ac3631f7fcdda7", "evidence_0/data_filter.json": "f07eef84b3081ad86bb5b48f810bbed569a76ee8e82cba9228beec908232d79e", "evidence_0/correctness_final.json": "d5a9fc3e02e6e5d204d4c5154c21d30dcf92aa7621620f1ea7f64e824a4ecfef", "evidence_0/correctness_initial.json": "cd6b7a551708c15a09099a58d7863fc0eee17e8b5ff36fd3ed2fdb7147ffbf3a", "evidence_0/best_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "evidence_0/manifest.json": "80dc3efef131f59bc7bb5005bc6d1de46350c405604dd3a710aa1f2d34c3762b", "training/initial_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/summary.json": "9519102ebab9faf3ef5881523d665b3974259891975a6b31c724206aa5f00237", "training/validation_step_000159_predictions.json": "6c0d4eee28bd23067354a845b9bf93f7a71445e8a9efec867f8c05a5361e6457", "training/correctness_final.json": "d5a9fc3e02e6e5d204d4c5154c21d30dcf92aa7621620f1ea7f64e824a4ecfef", "training/best_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/validation_step_000000_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/manifest.json": "5a0437bf2e3bc4422a2ff45e18e4431ba916ac682d28934e82e6c6bfa0208772" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read", "dataset_compatibility": { "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917", "reference_dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "protected_split_sha256": { "validation": "411199524c930d33fed8e1afa24597c59d400a500195fbec33d96afafd7ce74f", "calibration": "58fea4f180f16e5e0f2c9fd5f57487d3106415ec8e37bdd003e629ea944c51b4", "test": "ea61477a192d0a7174bcf0536f7664b547fdefe0bca9fe02c8e13c621e380819", "holdout": "0fb1bf6baf32374cf5dd8059428f058929884c903daa26be518ef14d33510cc3" }, "scope": "Only training data may differ; protected source bytes verified without reading labels or predictions" } }, "candidates": [ { "name": "gx10-4b", "host": "local", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b/training/best.pt", "checkpoint_sha256": "8956eb6c0cfbb02124aeefd99c3b418c55f55fdb9a64260350622d98dbba1aec", "step": 2500, "training_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "model_provenance": { "model_id": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "license": "apache-2.0" }, "data_signature": "c00527e505fba87aabbfa65ba5fb7a67cf628ac7d46d7598f270936a894e3884", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 8, "alpha": 16.0, "head_only": false, "two_pass": true, "lr": 0.0001, "head_lr": 0.0001, "seed": 431, "effective_batch": 4, "branch_batch_size": 1, "max_tokens": 512, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": null, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b/training/best_validation_predictions.json", "prediction_sha256": "16ff7745cdbfa53b87ccf2445796cf6a0c28d148a00e6c3926c72c36e75d058e", "metrics": { "macro_nll": 0.30729043330681044, "accuracy": 0.935546875, "count": 512, "per_family_nll": { "arc": 0.10747530600586384, "banking": 0.09334830709159572, "boolq": 0.6294207016878283, "snli": 0.3989174184419539 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.18864030037724333, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.18864030037724333, "raw_macro_nll": 0.30729043330681044, "accuracy": 0.935546875, "folds": [ { "fold": 0, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.17835820449058912, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 61, "temperature": 2.5292979964461435, "training_macro_nll": 0.17429139103918154, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.18674345787632526, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.20658826149875223, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.14837152412562352, "raw_macro_nll": 0.10747530600586384, "accuracy": 0.953125 }, "banking": { "count": 128, "score": 0.06337688481028368, "raw_macro_nll": 0.09334830709159572, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.3119620718470344, "raw_macro_nll": 0.6294207016878283, "accuracy": 0.890625 }, "snli": { "count": 128, "score": 0.23085072072603174, "raw_macro_nll": 0.3989174184419539, "accuracy": 0.921875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b/training/correctness_initial.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent" ], "evidence_sha256": { "evidence_0/resumed_initial_predictions.json": "e686d63b8f93d466dc085e4e32ab904b218839246dee8211e8b71cff512db985", "evidence_0/summary.json": "08fd0a87f22a96aa736a1e0f5eb52dd72b46b002a4da02518bc1420271aafe16", "evidence_0/data_filter.json": "3a514e34a8a6f7e35776bafe5b19015d6c5cec0eaabea5558aa896471aba3b56", "evidence_0/validation_step_000128_predictions.json": "e686d63b8f93d466dc085e4e32ab904b218839246dee8211e8b71cff512db985", "evidence_0/correctness_initial.json": "5a3a0b5b7cd5b9ac2840fe9a6de37d60fc069f1958e315a8a0c973799a960198", "evidence_0/best_validation_predictions.json": "e686d63b8f93d466dc085e4e32ab904b218839246dee8211e8b71cff512db985", "evidence_0/manifest.json": "b88894cedd20a2ae6dfeaf7c8f5e5b67b9556a118c6f4eb972a952555dc9c51d", "training/validation_step_002500_predictions.json": "16ff7745cdbfa53b87ccf2445796cf6a0c28d148a00e6c3926c72c36e75d058e", "training/validation_step_004000_predictions.json": "4d11441e85312b3633c35bc262f0c2fd76fe71b8836e71d9bf711e8b475ddeca", "training/resumed_initial_predictions.json": "9f1f8ccbc26a29ba0a0615ff1cbf278776c01c003b63176bae41df5a5bde92e6", "training/summary.json": "26cc23ba41a59d5875801d78fb693225736d1abc930a8a9662fe21c97b841464", "training/data_filter.json": "3a514e34a8a6f7e35776bafe5b19015d6c5cec0eaabea5558aa896471aba3b56", "training/validation_step_001000_predictions.json": "504c6675ae2d8e54b7633f79d5a597d856a332316200f31bb26a9019c7389bec", "training/validation_step_000178_predictions.json": "9f1f8ccbc26a29ba0a0615ff1cbf278776c01c003b63176bae41df5a5bde92e6", "training/validation_step_000500_predictions.json": "05bc0b72ce31dd4dda75f5e522c93808d0aae970fd3eb7b9b23a072b4f9391d9", "training/validation_step_000128_predictions.json": "e686d63b8f93d466dc085e4e32ab904b218839246dee8211e8b71cff512db985", "training/correctness_initial.json": "5a3a0b5b7cd5b9ac2840fe9a6de37d60fc069f1958e315a8a0c973799a960198", "training/best_validation_predictions.json": "16ff7745cdbfa53b87ccf2445796cf6a0c28d148a00e6c3926c72c36e75d058e", "training/manifest.json": "8c81e2582258ec2c554b284e355e8059f317d91d24ec775e32dc86dc696c5845", "training/validation_step_003500_predictions.json": "274faab9cf041bb184959f19ae8a591132747d733b187d6eb44e3f5c6905374c", "training/validation_step_003000_predictions.json": "3c40da8a7b688e74e225e2c28f6a8da6616b18274b897ff6d73df3b5bf6ac3ee", "training/validation_step_001500_predictions.json": "58e94a7adb55c1d0922de7583403c7ef82c89bf19b6e1582908f35e7c2bb3335", "training/validation_step_002000_predictions.json": "72aba24da717c7b0d2186657e290952052b10c054be472692aa7117420c7ff06" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read" }, { "name": "spark-a-4b-low-lr", "host": "andy@192.168.8.111", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260917T075209Z-spark-a-final-validation", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-a-4b-low-lr/training/best.pt", "checkpoint_sha256": "c63f1ac805aa6506c646c7c2ed8901a577ba69c42a98e64a1c03e2123de6142a", "step": 4000, "training_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "model_provenance": { "model_id": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "license": "apache-2.0" }, "data_signature": "c00527e505fba87aabbfa65ba5fb7a67cf628ac7d46d7598f270936a894e3884", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 8, "alpha": 16.0, "head_only": false, "two_pass": true, "lr": 3e-05, "head_lr": 3e-05, "seed": 431, "effective_batch": 4, "branch_batch_size": 1, "max_tokens": 512, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": 7500, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-a-4b-low-lr/training/best_validation_predictions.json", "prediction_sha256": "c8de91ece5e79c61c565c20b559da53b095752221bc37cde79bdee66f0ab53c4", "metrics": { "macro_nll": 0.2256861685108023, "accuracy": 0.9375, "count": 512, "per_family_nll": { "arc": 0.18791577746143256, "banking": 0.03133859366444661, "boolq": 0.4430523229150521, "snli": 0.240437980002278 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.19001961925646324, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.19001961925646324, "raw_macro_nll": 0.2256861685108023, "accuracy": 0.9375, "folds": [ { "fold": 0, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.18699174137845864, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.17795018696347403, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.17301629438038557, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 55, "temperature": 1.8407720014689555, "training_macro_nll": 0.21245712710318365, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.17338996945117396, "raw_macro_nll": 0.18791577746143256, "accuracy": 0.953125 }, "banking": { "count": 128, "score": 0.031567041855944666, "raw_macro_nll": 0.03133859366444661, "accuracy": 0.9921875 }, "boolq": { "count": 128, "score": 0.3262259595002021, "raw_macro_nll": 0.4430523229150521, "accuracy": 0.8828125 }, "snli": { "count": 128, "score": 0.2288955062185322, "raw_macro_nll": 0.240437980002278, "accuracy": 0.921875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-a-4b-low-lr/training/correctness_final.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts" ], "evidence_sha256": { "evidence_0/validation_step_000008_predictions.json": "d520b46127014161384c46d856fcfa167d4bb667a369235678f2e96ab0e54029", "evidence_0/initial_validation_predictions.json": "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b", "evidence_0/summary.json": "9bda8ed53ed42f7a8b4dea08acbb6c4c58635bf64dafa925b1a3575e9a4f89b4", "evidence_0/data_filter.json": "3a514e34a8a6f7e35776bafe5b19015d6c5cec0eaabea5558aa896471aba3b56", "evidence_0/correctness_final.json": "cb2649db1735dbcbc90f83d9f95520a7dd553471b7d0c88316205207ad66acb8", "evidence_0/correctness_initial.json": "5d6334a0c66ed567fa9709df4ea5b6040c88bb5258e864332e88be2bb9586634", "evidence_0/best_validation_predictions.json": "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b", "evidence_0/manifest.json": "18672b6c49227a08ba1303a4958a9fa08ae78b5fe59e30c99902e5d61ccaddf0", "training/validation_step_004000_predictions.json": "c8de91ece5e79c61c565c20b559da53b095752221bc37cde79bdee66f0ab53c4", "training/validation_step_004765_predictions.json": "4ec3fa00640f6a91eaec2f2f002a40e71b2aac0bf305789fc73dacb1a101930c", "training/summary.json": "5537b83636f7a22aa8a904bad488a19776a9fac921fdd51304309fac09c87707", "training/correctness_final.json": "cb2649db1735dbcbc90f83d9f95520a7dd553471b7d0c88316205207ad66acb8", "training/best_validation_predictions.json": "c8de91ece5e79c61c565c20b559da53b095752221bc37cde79bdee66f0ab53c4", "training/manifest.json": "74dcd5738d2cdcb53689517a4c9409e21bd3c355458cc165f4c0ac17b82289a2" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read" }, { "name": "spark-b-2b", "host": "andy@192.168.8.204", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-2b/training/best.pt", "checkpoint_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11", "step": 2000, "training_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "model_provenance": { "model_id": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "license": "apache-2.0" }, "data_signature": "dedef2b6406f252c165ee317925935079282f8a8ac96f034e89b07048d79f9e1", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 16, "alpha": 32.0, "head_only": false, "two_pass": false, "lr": 0.0001, "head_lr": 0.0001, "seed": 431, "effective_batch": 4, "branch_batch_size": 2, "max_tokens": 768, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": null, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-2b/training/best_validation_predictions.json", "prediction_sha256": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4", "metrics": { "macro_nll": 0.25676974955497256, "accuracy": 0.8984375, "count": 512, "per_family_nll": { "arc": 0.33955257651665527, "banking": 0.06537141992655102, "boolq": 0.34265219288051024, "snli": 0.2795028088961736 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.2552943737691716, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2552943737691716, "raw_macro_nll": 0.25676974955497256, "accuracy": 0.8984375, "folds": [ { "fold": 0, "temperature_index": 48, "temperature": 1.2705741052085413, "training_macro_nll": 0.27533710550694335, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.25585027399066257, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 45, "temperature": 1.0839269140212033, "training_macro_nll": 0.22881493804363362, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.24681503020373022, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.3379051798328397, "raw_macro_nll": 0.33955257651665527, "accuracy": 0.890625 }, "banking": { "count": 128, "score": 0.06793271948328157, "raw_macro_nll": 0.06537141992655102, "accuracy": 0.9609375 }, "boolq": { "count": 128, "score": 0.33084980282335186, "raw_macro_nll": 0.34265219288051024, "accuracy": 0.84375 }, "snli": { "count": 128, "score": 0.28448979293721316, "raw_macro_nll": 0.2795028088961736, "accuracy": 0.8984375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-2b/training/correctness_final.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts" ], "evidence_sha256": { "evidence_0/initial_validation_predictions.json": "4ee92802d7a8d642c8801dfd96f5bb18df80f745632fc1c8d99f9da9fe731c5b", "evidence_0/summary.json": "05cf399210326916c92e404ed815ca8644bd3db09fa027cbfe4686fac47a902f", "evidence_0/data_filter.json": "e60da06955fe9e8a70a3de1ddaeb261e53af8f3a5e1d13ee62932d3773a1be24", "evidence_0/correctness_final.json": "55e085c42e416d00fd4cdad9fb72d121f780fb9005226549ce7eaeddc9eea051", "evidence_0/correctness_initial.json": "7414b27fe26d2d52ea023ba5cc4d8bc9a46d9bf60a8d54358ad2ad803c03c26a", "evidence_0/validation_step_000040_predictions.json": "d44337f6e32bd8f1e9db01f533ce031dafec5a6162fe43d19dde40e0a3c39c75", "evidence_0/manifest.json": "e1efa2f4d350c678ba726e88dc85d8e7f3c8dc4e698742897892e578e5d94c58", "training/validation_step_002500_predictions.json": "7933d1b780889c9fd9313f618b1cbb6a4d257fffb3d57bc3f938d273f818f573", "training/validation_step_004000_predictions.json": "5379893699f950172b6708769e239c0f2df1525ac7ddd03e4e84a59f7ff8e765", "training/resumed_initial_predictions.json": "d44337f6e32bd8f1e9db01f533ce031dafec5a6162fe43d19dde40e0a3c39c75", "training/summary.json": "afa53acf8c241943acb1ad745db398f431ec7ffb8eef52bd75afc4c418067a08", "training/data_filter.json": "e60da06955fe9e8a70a3de1ddaeb261e53af8f3a5e1d13ee62932d3773a1be24", "training/validation_step_001000_predictions.json": "d821ad339fe52849c9a0e99a1a16e3b3598b08f8e5a6d2765584745001ebfbc3", "training/correctness_final.json": "dc12c9800ce68562e1fffe9d238170e7bd8a5f4400509dbb3a7207c2383c7522", "training/validation_step_000500_predictions.json": "d3e1d481a6efbc4077be0c307f884f2d0e603a45d315e9764397334d11b572c3", "training/correctness_initial.json": "39d23512e99743437b88b5b10110e501853976b5292581481b5d2a5b7f2c5cdd", "training/best_validation_predictions.json": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4", "training/validation_step_000040_predictions.json": "d44337f6e32bd8f1e9db01f533ce031dafec5a6162fe43d19dde40e0a3c39c75", "training/manifest.json": "e6f39dc5c797a0c1f5aa6ca176509d4cb1ef176a98af0bb6ee0946a5acbbc09e", "training/validation_step_004500_predictions.json": "26e8105d8ba1c87b286e6646fdc562b337dea563a21d6f10322e5f0e9a80c572", "training/validation_step_006000_predictions.json": "dca1b399bc43cbebcbe500f95b5d0b62209631b7e22a5a547cb4c00328904998", "training/validation_step_003500_predictions.json": "53a411e440cde4b8c30c9dacddbf1be28f7a72f8f8818b747bb04613f9f271c3", "training/validation_step_005500_predictions.json": "06ea55c0536e2f46e5225ab0fddf99a521f0ef4d6a5a8ba37a2aa794f55d6839", "training/validation_step_003000_predictions.json": "a2a2d5941dd23ef131d9d0654d4431c8e3ce0be0af66c4c481bf44d0636a0e95", "training/validation_step_005000_predictions.json": "c962bbdae92ac100d2eb88c2ecb5475e6d6fd12307dbedcd2a955e2691edc3a3", "training/validation_step_001500_predictions.json": "b80b912df7486e2f2d8f2f26d1a7c8ba12d1a4bde6d5b900a6f41e6eb9f6a5a7", "training/validation_step_002000_predictions.json": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read" }, { "name": "spark-b-4b-refinement", "host": "andy@192.168.8.204", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260917T075209Z-spark-b-final-validation", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-4b-refinement/training/best.pt", "checkpoint_sha256": "5f57ec38796d132edfa23638fbce66131fd4e7dfe87ceeadaba2b0e0d5c78024", "step": 1500, "training_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "model_provenance": { "model_id": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "license": "apache-2.0" }, "data_signature": "c00527e505fba87aabbfa65ba5fb7a67cf628ac7d46d7598f270936a894e3884", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 8, "alpha": 16.0, "head_only": false, "two_pass": true, "lr": 1e-05, "head_lr": 1e-05, "seed": 432, "effective_batch": 4, "branch_batch_size": 1, "max_tokens": 512, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": 5000, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-4b-refinement/training/best_validation_predictions.json", "prediction_sha256": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "metrics": { "macro_nll": 0.190872636672039, "accuracy": 0.947265625, "count": 512, "per_family_nll": { "arc": 0.1125077638524943, "banking": 0.05960886883339138, "boolq": 0.3498694938007437, "snli": 0.24150442020152654 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.1701497127614862, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.1701497127614862, "raw_macro_nll": 0.190872636672039, "accuracy": 0.947265625, "folds": [ { "fold": 0, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.1622086936723931, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15901522343009294, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15824280619091902, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.19176642571330008, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.13049231104529602, "raw_macro_nll": 0.1125077638524943, "accuracy": 0.96875 }, "banking": { "count": 128, "score": 0.05035731679828836, "raw_macro_nll": 0.05960886883339138, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.28226935283016463, "raw_macro_nll": 0.3498694938007437, "accuracy": 0.9296875 }, "snli": { "count": 128, "score": 0.21747987037219577, "raw_macro_nll": 0.24150442020152654, "accuracy": 0.9140625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/spark-b-4b-refinement/training/correctness_final.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts" ], "evidence_sha256": { "evidence_0/validation_step_000008_predictions.json": "b2a477b919ef21569299b4adf3b8c6d78806659e190dee622fbf393b16d79905", "evidence_0/initial_validation_predictions.json": "16ff7745cdbfa53b87ccf2445796cf6a0c28d148a00e6c3926c72c36e75d058e", "evidence_0/summary.json": "929d4851d5b829a4cd69d337ab858f4f78480d6a654968d83141583c846fcec4", "evidence_0/data_filter.json": "3a514e34a8a6f7e35776bafe5b19015d6c5cec0eaabea5558aa896471aba3b56", "evidence_0/correctness_final.json": "f9ad8b0308a09577abc390412f214cc6289b1108ced3907570c6d4cf58539158", "evidence_0/correctness_initial.json": "fd1b0d769f7a3f1cddacf00c19b0e3cec76c97c1d1e83a434ec2950bf178c677", "evidence_0/best_validation_predictions.json": "16ff7745cdbfa53b87ccf2445796cf6a0c28d148a00e6c3926c72c36e75d058e", "evidence_0/manifest.json": "e1ac6aaf86db2fff6c007a1018503877976ba04de7415125594142158b6f11a5", "training/summary.json": "cd27a9db9af2c3aa93f7c5c1dbb22108a731bc77bddce6fdc96e5801866c0e44", "training/correctness_final.json": "f9ad8b0308a09577abc390412f214cc6289b1108ced3907570c6d4cf58539158", "training/best_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/manifest.json": "caca473b9a28172dc6c6e0b1b25053d7a2af459cf2abc77e9a21fa179cff471c", "training/validation_step_001500_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/validation_step_002000_predictions.json": "717d3cc8f43c62b7bcde6a3c040dfbc9caaa7195be116bdbbc98acf43b78d4bb" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read" }, { "name": "gx10-4b-expanded", "host": "local", "eligible": true, "source_campaign": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h", "source_training": "/home/andy/ai/opensysone/runs/20260917T075209Z-gx10-final-validation", "checkpoint": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/training/best.pt", "checkpoint_sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca", "step": 0, "training_source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86", "model_provenance": { "model_id": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "license": "apache-2.0" }, "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917", "output": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts", "resume": null, "warm_start": "/home/andy/ai/opensysone/runs/20260917T070415Z-expanded-parent/best.pt", "allow_train_data_change": true, "steps": 8, "epochs": 3, "rank": 8, "alpha": 16.0, "head_only": false, "two_pass": true, "lr": 2e-05, "head_lr": 2e-05, "seed": 433, "effective_batch": 4, "branch_batch_size": 1, "max_tokens": 512, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00Z", "schedule_steps": 3500, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "prediction_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/evidence_0/best_validation_predictions.json", "prediction_sha256": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "metrics": { "macro_nll": 0.190872636672039, "accuracy": 0.947265625, "count": 512, "per_family_nll": { "arc": 0.1125077638524943, "banking": 0.05960886883339138, "boolq": 0.3498694938007437, "snli": 0.24150442020152654 }, "selection_metric": "crossfit_temperature_nll_v1", "selection_score": 0.1701497127614862, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.1701497127614862, "raw_macro_nll": 0.190872636672039, "accuracy": 0.947265625, "folds": [ { "fold": 0, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.1622086936723931, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15901522343009294, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.15824280619091902, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.19176642571330008, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.13049231104529602, "raw_macro_nll": 0.1125077638524943, "accuracy": 0.96875 }, "banking": { "count": 128, "score": 0.05035731679828836, "raw_macro_nll": 0.05960886883339138, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.28226935283016463, "raw_macro_nll": 0.3498694938007437, "accuracy": 0.9296875 }, "snli": { "count": 128, "score": 0.21747987037219577, "raw_macro_nll": 0.24150442020152654, "accuracy": 0.9140625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } } }, "correctness_path": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection_attempt_20260917T080731899139Z/gx10-4b-expanded/training/correctness_final.json", "source_evidence_dirs": [ "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts" ], "evidence_sha256": { "evidence_0/validation_step_000008_predictions.json": "8f0037152e67aebba05db28024bf02fc7fcb84986795efd464d171c2bc4dddcf", "evidence_0/initial_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "evidence_0/summary.json": "bc5e3ad91ecf3ef13ed3b82e408ebdd7110102a924f8b046e5ac3631f7fcdda7", "evidence_0/data_filter.json": "f07eef84b3081ad86bb5b48f810bbed569a76ee8e82cba9228beec908232d79e", "evidence_0/correctness_final.json": "d5a9fc3e02e6e5d204d4c5154c21d30dcf92aa7621620f1ea7f64e824a4ecfef", "evidence_0/correctness_initial.json": "cd6b7a551708c15a09099a58d7863fc0eee17e8b5ff36fd3ed2fdb7147ffbf3a", "evidence_0/best_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "evidence_0/manifest.json": "80dc3efef131f59bc7bb5005bc6d1de46350c405604dd3a710aa1f2d34c3762b", "training/initial_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/summary.json": "9519102ebab9faf3ef5881523d665b3974259891975a6b31c724206aa5f00237", "training/validation_step_000159_predictions.json": "6c0d4eee28bd23067354a845b9bf93f7a71445e8a9efec867f8c05a5361e6457", "training/correctness_final.json": "d5a9fc3e02e6e5d204d4c5154c21d30dcf92aa7621620f1ea7f64e824a4ecfef", "training/best_validation_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/validation_step_000000_predictions.json": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c", "training/manifest.json": "5a0437bf2e3bc4422a2ff45e18e4431ba916ac682d28934e82e6c6bfa0208772" }, "selection_scope": "Fixed four-fold temperature-crossfit validation macro-family NLL; no reserved calibration/test/holdout predictions read", "dataset_compatibility": { "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917", "reference_dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "protected_split_sha256": { "validation": "411199524c930d33fed8e1afa24597c59d400a500195fbec33d96afafd7ce74f", "calibration": "58fea4f180f16e5e0f2c9fd5f57487d3106415ec8e37bdd003e629ea944c51b4", "test": "ea61477a192d0a7174bcf0536f7664b547fdefe0bca9fe02c8e13c621e380819", "holdout": "0fb1bf6baf32374cf5dd8059428f058929884c903daa26be518ef14d33510cc3" }, "scope": "Only training data may differ; protected source bytes verified without reading labels or predictions" } } ], "selected_utc": "2026-09-17T08:09:51.199087+00:00", "reference_sha256": "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b" }